Stock Data Collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create CSVs for top 10 stocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in packages\n",
    "from pandas_datareader import data, wb\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "from itertools import chain\n",
    "import matplotlib.pyplot as plt\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "import math, time\n",
    "import itertools\n",
    "from sklearn import preprocessing\n",
    "from datetime import datetime\n",
    "from operator import itemgetter\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.layers.recurrent import LSTM\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "month_dict = {'01': 'Jan', '02': 'Feb', '03': 'Mar', '04': 'Apr', '05': 'May', '06': 'Jun', \n",
    "             '07': 'Jul', '08': 'Aug', '09': 'Sep', '10': 'Oct', '11': 'Nov', '12': 'Dec'}\n",
    "\n",
    "def get_stock_data(ticker, start, end):\n",
    "    \"\"\"\n",
    "    Input: stock ticker, start date, end date\n",
    "    Output: numpy array of stock data\n",
    "    \"\"\"\n",
    "    \n",
    "    # Function to check if date is valid\n",
    "    def check_date(date):\n",
    "        return len(date) == 3 and len(date[0]) == 4 and len(date[1]) == 2 and len(date[2]) == 2\n",
    "    \n",
    "    # Format start and end days and check validity of dates\n",
    "    start = start.split('-')\n",
    "    end = end.split('-')\n",
    "    errors = not (check_date(start) and check_date(end))\n",
    "    if errors:\n",
    "        print 'Your dates need to be of the form \\'YYYY-MM-DD\\'!'\n",
    "        return\n",
    "    start_date = month_dict[start[1]] + \"+\" + start[2] + \"%2C+\" + start[0]\n",
    "    end_date = month_dict[end[1]] + \"+\" + end[2] + \"%2C+\" + end[0]\n",
    "    \n",
    "    start_dt = datetime(int(start[0]), int(start[1]), int(start[2]))\n",
    "    end_dt = datetime(int(end[0]), int(end[1]), int(end[2]))\n",
    "    timelapse = end_dt - start_dt\n",
    "    business_days = np.busday_count(start_dt, end_dt)\n",
    "#     print business_days, \"business days\"    \n",
    "    \n",
    "    url = \"http://finance.google.com/finance/historical?q=\" + ticker + \"&startdate=\" + start_date + \"&enddate=\" + end_date + \"&num=30&ei=SxDHWfiBHMzYjAGLnrr4Ag&output=csv\"\n",
    "    col_names = ['Date','Open','High','Low','Close','Volume']\n",
    "    data = pd.read_csv(url, header=0, names=col_names)\n",
    "#     print data.shape\n",
    "#     data = np.array(data)[::-1] # order rows in chronological order\n",
    "#     data = data.astype(float)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2979, 6)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = get_stock_data('AAPL', '2006-01-01', '2017-11-01')\n",
    "# x.to_csv('data/test.csv')\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3-Nov-17</td>\n",
       "      <td>174.00</td>\n",
       "      <td>174.26</td>\n",
       "      <td>171.12</td>\n",
       "      <td>172.50</td>\n",
       "      <td>59398631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2-Nov-17</td>\n",
       "      <td>166.60</td>\n",
       "      <td>168.50</td>\n",
       "      <td>165.28</td>\n",
       "      <td>168.11</td>\n",
       "      <td>41393373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1-Nov-17</td>\n",
       "      <td>169.87</td>\n",
       "      <td>169.94</td>\n",
       "      <td>165.61</td>\n",
       "      <td>166.89</td>\n",
       "      <td>33637762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31-Oct-17</td>\n",
       "      <td>167.90</td>\n",
       "      <td>169.65</td>\n",
       "      <td>166.94</td>\n",
       "      <td>169.04</td>\n",
       "      <td>36046828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30-Oct-17</td>\n",
       "      <td>163.89</td>\n",
       "      <td>168.07</td>\n",
       "      <td>163.72</td>\n",
       "      <td>166.72</td>\n",
       "      <td>44700772</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date    Open    High     Low   Close    Volume\n",
       "0   3-Nov-17  174.00  174.26  171.12  172.50  59398631\n",
       "1   2-Nov-17  166.60  168.50  165.28  168.11  41393373\n",
       "2   1-Nov-17  169.87  169.94  165.61  166.89  33637762\n",
       "3  31-Oct-17  167.90  169.65  166.94  169.04  36046828\n",
       "4  30-Oct-17  163.89  168.07  163.72  166.72  44700772"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/MMM_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/AXP_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/AAPL_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/BA_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/CAT_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/CVX_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/CSCO_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/KO_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/DIS_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/XOM_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/GE_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/GS_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/HD_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/IBM_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/INTC_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/JNJ_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/JPM_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/MCD_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/MRK_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/MSFT_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/NKE_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/PFE_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/PG_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/TRV_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/UTX_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/UNH_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/VZ_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/V_2006-01-01_to_2017-11-01.csv\n",
      "(2425, 6)\n",
      "data/WMT_2006-01-01_to_2017-11-01.csv\n",
      "(2980, 6)\n",
      "data/GOOGL_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/AMZN_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n",
      "data/AABA_2006-01-01_to_2017-11-01.csv\n",
      "(2979, 6)\n"
     ]
    }
   ],
   "source": [
    "stocks = ['MMM', 'AXP', 'AAPL', 'BA', 'CAT', 'CVX', 'CSCO', 'KO', 'DIS', 'XOM', 'GE',\n",
    "          'GS', 'HD', 'IBM', 'INTC', 'JNJ', 'JPM', 'MCD', 'MRK', 'MSFT', 'NKE', 'PFE',\n",
    "          'PG', 'TRV', 'UTX', 'UNH', 'VZ', 'V', 'WMT', 'GOOGL', 'AMZN', 'AABA']\n",
    "\n",
    "start_date = '2006-01-01'\n",
    "end_date = '2017-11-01'\n",
    "\n",
    "for ticker in stocks:\n",
    "    file_name = 'data/' + ticker + '_' + start_date + '_to_' + end_date + '.csv'\n",
    "    print file_name\n",
    "    data = get_stock_data(ticker, start_date, end_date)\n",
    "    print data.shape\n",
    "    data.to_csv(file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stocks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}