05 AllFilesSummaryStatistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ASC Accelerometer analysis via Summary Statistics \n",
    "\n",
    "FDeveloping moving average acceleration and moving Standard Deviation for each data set.\n",
    "\n",
    "Can run through a whole folder, or through random selections from the folder\n",
    "\n",
    "Currently brute Force Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Standard Header used on the projects\n",
    "\n",
    "#first the major packages used for math and graphing\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from cycler import cycler\n",
    "import scipy.special as sp\n",
    "import pandas as pd\n",
    "\n",
    "#Custome graph format style sheet\n",
    "plt.style.use('Prospectus.mplstyle')\n",
    "\n",
    "#If being run by a seperate file, use the seperate file's graph format and saving paramaeters\n",
    "#otherwise set what is needed\n",
    "if not 'Saving' in locals():\n",
    "    Saving = False\n",
    "if not 'Titles' in locals():\n",
    "    Titles = True\n",
    "if not 'Ledgends' in locals():\n",
    "    Ledgends = True\n",
    "if not 'FFormat' in locals():\n",
    "    FFormat = '.png'\n",
    "if not 'location' in locals():\n",
    "    #save location.  First one is for running on home PC, second for running on the work laptop.  May need to make a global change\n",
    "    #location = 'E:\\\\Documents\\\\Dan\\\\Code\\\\Prospectus\\\\Document\\\\Figures\\\\'\n",
    "    #location = 'C:\\\\Users\\\\dhendrickson\\\\Documents\\\\github\\\\FigsAndPlots\\\\FigsAndPlotsDocument\\\\Figures\\\\'\n",
    "    location = 'E:\\\\Documents\\\\Dan\\\\Phd\\\\Play\\\\'\n",
    "\n",
    "#Standard cycle for collors and line styles\n",
    "default_cycler = (cycler('color', ['0.00', '0.40', '0.60', '0.70']) + cycler(linestyle=['-', '--', ':', '-.']))\n",
    "plt.rc('axes', prop_cycle=default_cycler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extra Headers:\n",
    "import DWFT as fp\n",
    "import os as os\n",
    "import pywt as py\n",
    "import statistics as st\n",
    "import os as os\n",
    "import pandas as pd\n",
    "import random\n",
    "import multiprocessing\n",
    "from joblib import Parallel, delayed\n",
    "from pywt._extensions._pywt import (DiscreteContinuousWavelet, ContinuousWavelet,\n",
    "                                Wavelet, _check_dtype)\n",
    "from pywt._functions import integrate_wavelet, scale2frequency\n",
    "import time\n",
    "\n",
    "my_cmap = plt.get_cmap('gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DataSet = np.genfromtxt(open('./Data/Jeep SD 10.txt','r'), delimiter=',',skip_header=4)\n",
    "Header = np.array(['T', 'X','Y','Z','R','Theta','Phi'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAcceleration(FileName):\n",
    "    try:\n",
    "        DataSet = np.genfromtxt(open(FileName,'r'), delimiter=',',skip_header=0)\n",
    "        if FileName[-25:-20] == \"Point\":\n",
    "            return [[FileName[61:],'x',DataSet[:,2]],[FileName[61:],'y',DataSet[:,3]],[FileName[61:],'z',DataSet[:,4]]]\n",
    "        else:\n",
    "            return [False,FileName,False]\n",
    "    except:\n",
    "        return [False,FileName,False]\n",
    "\n",
    "\n",
    "def makeStats(DataArray,Arange=50):\n",
    "    try:\n",
    "        #Arange = 50\n",
    "        length = np.shape(DataArray[2])[0]\n",
    "        StdDev = np.zeros(length)\n",
    "        for j in range(length-Arange):\n",
    "            k = (length-1)-j\n",
    "            DataArray[2][k] = np.average(DataArray[2][k-Arange:k])\n",
    "            StdDev[k]=st.stdev(DataArray[2][k-Arange:k])\n",
    "        return [DataArray[0],DataArray[1],max(DataArray[2]),max(StdDev)]\n",
    "    except:\n",
    "        return ['','','','']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:1970: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  result = asarray(a).shape\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 of 1 Time: 2.9684828519821167\n"
     ]
    }
   ],
   "source": [
    "num_cores = multiprocessing.cpu_count()-1\n",
    "#folder1 = 'C:\\\\Users\\\\Hendrickson\\\\Desktop\\\\Phone Acceleration\\\\'\n",
    "#folder2 = 'E:\\\\Documents\\\\Dan\\\\PhD\\\\Data Backup\\\\ASC Accel Pi\\\\Excel Versions\\\\'\n",
    "folder1 = \"D:\\\\\"\n",
    "folder2 = \"D:\\\\SmallCopy\\\\\"\n",
    "files = os.listdir(folder2)\n",
    "\n",
    "GroupSize = 25\n",
    "if np.shape(files)[0] < GroupSize:\n",
    "    GroupSize = np.shape(files)[0]\n",
    "\n",
    "Groups = 0\n",
    "\n",
    "if Groups !=0:\n",
    "    files = random.sample(files,GroupSize*Groups-1)\n",
    "\n",
    "loops = int(float(np.size(files))/float(GroupSize))+1\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    AllEvents=[]\n",
    "    Fails = []\n",
    "    for i in range(loops):\n",
    "        \n",
    "        AllAccels = Parallel(n_jobs=num_cores)(delayed(getAcceleration)(folder2+file) for file in files[i*GroupSize:((i+1)*GroupSize)])\n",
    "        Flattened = []\n",
    "        for j in range(np.shape(AllAccels)[0]):\n",
    "            if AllAccels[j][0] == False :\n",
    "                if AllAccels[j][1][4:9] == 'Point':\n",
    "                    print(j,AllAccels[j][1])\n",
    "            else: \n",
    "                for k in range(3):\n",
    "                    Flattened.append(AllAccels[j][k])\n",
    "        Events =  Parallel(n_jobs=num_cores)(delayed(makeStats)(DataArrays) for DataArrays in Flattened)\n",
    "        Events = np.matrix(Events)\n",
    "        if np.size(AllEvents) == 0:\n",
    "            AllEvents=Events\n",
    "        else:\n",
    "            try:\n",
    "                AllEvents=np.concatenate((AllEvents,Events), axis=0)\n",
    "            except:\n",
    "                Fails.append(Events)\n",
    "        print(str(i+1)+' of '+str(loops),'Time: '+ str((time.time()-start)/60.0))\n",
    "    \n",
    "    df = pd.DataFrame(data=AllEvents)\n",
    "    df.to_csv(folder1 + 'StatisticsReport.csv', sep=',', index = False, header=False,quotechar='\"')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Point\n"
     ]
    }
   ],
   "source": [
    "File = folder2+'60kPoints-210709-0820-s1.csv'\n",
    "print(File[-25:-20])"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b3ba2566441a7c06988d0923437866b63cedc61552a5af99d1f4fb67d367b25f"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}