consolidated_wms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib\n",
    "import urllib2\n",
    "import re\n",
    "import json\n",
    "import pandas as pd\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "from collections import defaultdict\n",
    "from bs4 import BeautifulSoup\n",
    "from scrapy import Selector\n",
    "from scrapy.selector import HtmlXPathSelector\n",
    "from weather import Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "search_text = \"bangalore\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>Yahoo Weather Information</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('1.Yahoo', 27, 26, 26, 27, 26, 26)\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    weather = Weather()\n",
    "    location = weather.lookup_by_location(search_text)\n",
    "    condition = location.condition()\n",
    "    yahoo_data_now = (\"Yahoo\", int((int(condition['temp'])-32)*float(0.5556)), \"\", \"\", \"\")\n",
    "\n",
    "    yahoo_url = \"https://query.yahooapis.com/v1/public/yql?\"\n",
    "    yql_query = \"select * from weather.forecast where woeid in (select woeid from geo.places(1) where text='{}')\".format(search_text)\n",
    "    yql_url = yahoo_url + urllib.urlencode({'q':yql_query}) + \"&format=json\"\n",
    "    result = urllib2.urlopen(yql_url).read()\n",
    "    data = json.loads(result)\n",
    "\n",
    "    week_data = data['query']['results'][\"channel\"][\"item\"][\"forecast\"]\n",
    "    day_1 = int((int(week_data[1]['high']) - 32) * 0.5556)\n",
    "    day_2 = int((int(week_data[2]['high']) - 32) * 0.5556)\n",
    "    day_3 = int((int(week_data[3]['high']) - 32) * 0.5556)\n",
    "    day_4 = int((int(week_data[4]['high']) - 32) * 0.5556)\n",
    "    day_5 = int((int(week_data[5]['high']) - 32) * 0.5556)\n",
    "    yahoo_data = (\"1.Yahoo\", int((int(condition['temp'])-32)*float(0.5556)), day_1, day_2, day_3, day_4, day_5)\n",
    "except:\n",
    "    yahoo_data = (\n",
    "        \"1.Yahoo\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "    )\n",
    "print yahoo_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>The Weather Network</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('2.Weather Network', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA')\n"
     ]
    }
   ],
   "source": [
    "base_url = \"https://www.theweathernetwork.com/in/api/location/search?searchText={}\".format(search_text)\n",
    "req = requests.get(base_url)\n",
    "\n",
    "data = req.json()\n",
    "data_url = []\n",
    "if data:\n",
    "    for i in data:\n",
    "        if i[\"name\"] == search_text.title() and i[\"country\"] == \"India\":\n",
    "            each_url = \"https://www.theweathernetwork.com{}\".format(i[\"url\"])            \n",
    "            data_url.append({\"each_url\": each_url, \"code\": i[\"code\"]})\n",
    "\n",
    "try:            \n",
    "    if data_url:\n",
    "    \n",
    "        req_url = data_url[0]\n",
    "        current_weather_url = \"https://www.theweathernetwork.com/in/api/savedlocation/index/?placecodes={}\".format(req_url[\"code\"])\n",
    "        current_weather_req = requests.get(current_weather_url).json()\n",
    "        current_day_temp = current_weather_req[0][\"temperature\"]\n",
    "    \n",
    "        resp = urllib2.urlopen(req_url[\"each_url\"]).read()\n",
    "        soup = BeautifulSoup(resp, \"lxml\")\n",
    "    \n",
    "        current_data_day_1 = soup.find_all(\"div\", attrs={\"class\": \"day_1\"})\n",
    "        body = \"\"\"{}\"\"\".format(current_data_day_1[0])\n",
    "        selection_day_1 = Selector(text=body).xpath(\"//div\").extract()\n",
    "        temp_day_1 = re.findall(\"\\d+\", selection_day_1[5])[0]\n",
    "\n",
    "        current_data_day_2 = soup.find_all(\"div\", attrs={\"class\": \"day_2\"})\n",
    "        body = \"\"\"{}\"\"\".format(current_data_day_2[0])\n",
    "        selection_day_2 = Selector(text=body).xpath(\"//div\").extract()\n",
    "        temp_day_2 = re.findall(\"\\d+\", selection_day_2[5])[0]\n",
    "\n",
    "        current_data_day_3 = soup.find_all(\"div\", attrs={\"class\": \"day_3\"})\n",
    "        body = \"\"\"{}\"\"\".format(current_data_day_3[0])\n",
    "        selection_day_3 = Selector(text=body).xpath(\"//div\").extract()\n",
    "        temp_day_3 = re.findall(\"\\d+\", selection_day_3[5])[0]\n",
    "\n",
    "        current_data_day_4 = soup.find_all(\"div\", attrs={\"class\": \"day_4\"})\n",
    "        body = \"\"\"{}\"\"\".format(current_data_day_4[0])\n",
    "        selection_day_4 = Selector(text=body).xpath(\"//div\").extract()\n",
    "        temp_day_4 = re.findall(\"\\d+\", selection_day_4[5])[0]\n",
    "\n",
    "        current_data_day_5 = soup.find_all(\"div\", attrs={\"class\": \"day_5\"})\n",
    "        body = \"\"\"{}\"\"\".format(current_data_day_5[0])\n",
    "        selection_day_5 = Selector(text=body).xpath(\"//div\").extract()\n",
    "        temp_day_5 = re.findall(\"\\d+\", selection_day_5[5])[0]\n",
    "\n",
    "        weather_network_data = (\n",
    "            \"2.Weather Network\",\n",
    "            current_day_temp,\n",
    "            temp_day_1,\n",
    "            temp_day_2,\n",
    "            temp_day_3,\n",
    "            temp_day_4,\n",
    "            temp_day_5,\n",
    "        )\n",
    "    else:\n",
    "        weather_network_data = (\n",
    "            \"2.Weather Network\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "        )\n",
    "except:\n",
    "        weather_network_data = (\n",
    "            \"2.Weather Network\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "            \"NA\",\n",
    "        )\n",
    "print weather_network_data  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4> MSN Weather </h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('3.MSN Weather', 27, 26, 27, 27, 'NA', 'NA')\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    base_url = \"http://weather.service.msn.com/data.aspx?weasearchstr={}&culture=en-US&weadegreetype=F&src=outlook\".format(search_text)\n",
    "    req = requests.get(base_url)\n",
    "    root = ET.fromstring(req.content)\n",
    "    data = root.__dict__['_children'][0].__dict__['_children'][0].__dict__['attrib']\n",
    "    data_1 = root.__dict__['_children'][0].__dict__[\"_children\"][3].__dict__['attrib']['high']\n",
    "    data_2 = root.__dict__['_children'][0].__dict__[\"_children\"][4].__dict__['attrib']['high']\n",
    "    data_3 = root.__dict__['_children'][0].__dict__[\"_children\"][5].__dict__['attrib']['high']\n",
    "    msn_weather_data = (\n",
    "        \"3.MSN Weather\",\n",
    "        int((int(data['temperature'])-32)*float(0.5556)),\n",
    "        int((int(data_1)-32)*float(0.5556)),\n",
    "        int((int(data_2)-32)*float(0.5556)),\n",
    "        int((int(data_3)-32)*float(0.5556)),\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "    )\n",
    "except Exception as e:\n",
    "    msn_weather_data = (\n",
    "        \"3.MSN Weather\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "    )\n",
    "print msn_weather_data  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4> Accurate Weather </h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('4.Accurate Weather', '28', '28', '28', '27', 'NA', 'NA')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:26: ScrapyDeprecationWarning: scrapy.selector.HtmlXPathSelector is deprecated, instantiate scrapy.Selector instead.\n",
      "/usr/local/lib/python2.7/dist-packages/parsel/selector.py:234: ScrapyDeprecationWarning: scrapy.selector.HtmlXPathSelector is deprecated, instantiate scrapy.Selector instead.\n",
      "  for x in result]\n"
     ]
    }
   ],
   "source": [
    "search_url = \"https://www.accuweather.com/en/search-locations\"\n",
    "search_data = {\"s\": search_text.title()}\n",
    "try:\n",
    "    req = requests.post(search_url, search_data)\n",
    "    resp = req.__dict__['_content']\n",
    "\n",
    "    soup = BeautifulSoup(resp, \"lxml\")\n",
    "    data_input_box = soup.find_all(\"input\", attrs={'id': 's'})\n",
    "\n",
    "    data_str = str(data_input_box[0])\n",
    "    data_url = re.findall(\"(https://www.accuweather.com/\\w+/\\w+/\\w+/[0-9]+/[\\w-]+/[0-9]+)\\\"\", data_str)[0]\n",
    "\n",
    "    base_url = data_url.replace(\"weather-forecast\", \"current-weather\")\n",
    "    base_link_url = base_url.replace(\"current-weather\", \"daily-weather-forecast\")\n",
    "    base_links = [\"{}?day={}\".format(base_link_url, i) for i in range(1, 6)]\n",
    "\n",
    "    req = urllib2.Request(base_url)\n",
    "    read_url = urllib2.urlopen(req)\n",
    "    response = read_url.read()\n",
    "    soup = BeautifulSoup(response, \"lxml\")\n",
    "    five_day_data = soup.find_all(\"li\", attrs={'data-href': base_links})\n",
    "\n",
    "    column_header = ('Day', 'Date', 'real_feel', 'precipitation', 'description')\n",
    "    data = []\n",
    "    for day in five_day_data:\n",
    "        hxs = HtmlXPathSelector(day).xpath(\"/html/body/p/text()\").extract()\n",
    "        doc = \"\"\"{}\"\"\".format(hxs[0].encode(\"utf-8\"))\n",
    "        each_day = re.sub(\"\\n+\",\n",
    "                          \",\",\n",
    "                          doc.decode(\"unicode_escape\").encode(\"ascii\", \"ignore\").replace(\"/\", \"\").replace(\"More\", \"\")\n",
    "                         )\n",
    "        data.append(filter(None, tuple(each_day.split(\",\")))) \n",
    "    accurate_weather_data = (\"4.Accurate Weather\", data[0][2], data[1][2], data[3][2], data[4][2], \"NA\", \"NA\")\n",
    "except:\n",
    "    accurate_weather_data = (\n",
    "        \"4.Accurate Weather\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "        \"NA\",\n",
    "    )\n",
    "print accurate_weather_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>Weather Bug Weather</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "base_url = \"https://weather.weatherbug.com/api/localities/search?query={}\".format(search_text)\n",
    "slug_data = requests.get(base_url).json()[0]['SlugName']\n",
    "data_url = \"https://weather.weatherbug.com/weather-forecast/10-day-weather/{}\".format(slug_data)\n",
    "\n",
    "data_req = urllib2.urlopen(data_url)\n",
    "response = data_req.read()\n",
    "soup = BeautifulSoup(response, \"lxml\")\n",
    "day_data = soup.find_all(\"div\", attrs={\"class\": \"day-card__desktop\"})\n",
    "temp_value = []\n",
    "for each in day_data:\n",
    "    body = \"\"\"{}\"\"\".format(each)\n",
    "    body = body.decode(\"utf-8\").encode('ascii','ignore').replace(\"\\n\", \"\").replace(\"\\r\", \"\")\n",
    "    select = Selector(text=body).xpath('//div[@class=\"temp\"]/text()').extract()\n",
    "    temp = int(round((int(select[0].strip()) - 32) * float(0.5556)))\n",
    "    temp_value.append(temp)\n",
    "weather_bug_data = (\"5.Weather Bug\", temp_value[0], temp_value[1], temp_value[2], temp_value[3], temp_value[4], temp_value[5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>BBC Weather</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('6.BBC Weather', 19, 19, 19, 20, 20, 'NA')\n"
     ]
    }
   ],
   "source": [
    "search_url = \"http://www.bbc.com/locator/default/en-GB/autocomplete.json?search={}&filter=international\".format(search_text)\n",
    "\n",
    "city_data = requests.get(search_url).json()\n",
    "try:\n",
    "    weather_data = []\n",
    "    if city_data:\n",
    "        city_data = city_data[0]\n",
    "        city_id = city_data['id']\n",
    "    \n",
    "        weather_url = \"http://www.bbc.com/weather/{}\".format(city_id)\n",
    "        data_req = urllib2.urlopen(weather_url)\n",
    "        response = data_req.read()\n",
    "        soup = BeautifulSoup(response, \"lxml\")\n",
    "        day_data = soup.find_all(\"td\", attrs={\"class\": \"min-temp\"})\n",
    "        for each in day_data:\n",
    "            body = \"\"\"{}\"\"\".format(each)\n",
    "            body = body.decode(\"utf-8\").encode('ascii','ignore')\n",
    "            select = Selector(text=body).xpath('//span[@data-unit=\"c\"]/text()').extract()\n",
    "            weather_data.append(int(select[0]))\n",
    "        bbc_weather_data = (\"6.BBC Weather\", weather_data[0], weather_data[1], weather_data[2], weather_data[3], weather_data[4], \"NA\")\n",
    "    else:\n",
    "        bbc_weather_data = (\"6.BBC Weather\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\")\n",
    "except Exception as e:\n",
    "    print e.message\n",
    "    bbc_weather_data = (\"6.BBC Weather\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\")\n",
    "print bbc_weather_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>The Weather Channel</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('7.The Weather Channel', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA')\n"
     ]
    }
   ],
   "source": [
    "search_url = \"https://dsx.weather.com/x/v2/web/loc/en_US/1/4/5/9/11/13/19/21/1000/1001/1003/us^/{}?format=json&pg=0%2C10\".format(search_text)\n",
    "\n",
    "try:\n",
    "    resp = requests.get(search_url).json()\n",
    "    data_url = \"https://weather.com/en-IN/weather/tenday/l/{}\".format(resp[0]['key'])\n",
    "\n",
    "    data_req = urllib2.urlopen(data_url)\n",
    "    response = data_req.read()\n",
    "    soup = BeautifulSoup(response, \"lxml\")\n",
    "    day_data = soup.find_all(\"td\", attrs={\"class\": \"temp\"})\n",
    "    weather_data = []\n",
    "    for each in day_data:\n",
    "        body = \"\"\"{}\"\"\".format(each)\n",
    "        body = body.decode(\"utf-8\").encode('ascii','ignore')\n",
    "        select = Selector(text=body).xpath('//div/span/text()').extract()\n",
    "        weather_data.append(int(select[0]))\n",
    "    weather_channel_data = (\"7.The Weather Channel\", weather_data[0], weather_data[1], weather_data[2], weather_data[3], weather_data[4], weather_data[5])\n",
    "except:\n",
    "    weather_channel_data = (\"7.The Weather Channel\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\")\n",
    "print weather_channel_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h4>The Skymet Weather</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('8.The Skymet Weather', u'27', u'26', u'27', u'28', u'27', u'28')\n"
     ]
    }
   ],
   "source": [
    "search_url = \"https://www.skymetweather.com/external/searchlocation?q={}&l=en&c=internal\".format(search_text)\n",
    "\n",
    "try:\n",
    "    resp = requests.get(search_url).json()\n",
    "    data_url = resp[0]['value']\n",
    "\n",
    "    data_req = urllib2.urlopen(data_url)\n",
    "    response = data_req.read()\n",
    "    soup = BeautifulSoup(response, \"lxml\")\n",
    "    day_data = soup.find_all(\"div\", attrs={\"class\": \"Forecast7Day\"})\n",
    "    body = \"\"\"{}\"\"\".format(day_data[0])\n",
    "    body = body.decode(\"utf-8\").encode('ascii','ignore')\n",
    "    select = Selector(text=body).xpath('//span[@class=\"maxt\"]/span/span[@class=\"c\"]/text()').extract()\n",
    "    skymet_weather_data = (\"8.The Skymet Weather\", select[0], select[1], select[2], select[3], select[4], select[5])\n",
    "except:\n",
    "    skymet_weather_data = (\"8.The Skymet Weather\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\", \"NA\")\n",
    "print skymet_weather_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h2>Consolidated Weather Data</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bangalore\n",
      "\n",
      "                      Current Weather Day_1 Day_2 Day_3 Day_4 Day_5\n",
      "Sites                                                              \n",
      "1.Yahoo                            27    26    26    27    26    26\n",
      "3.MSN Weather                      27    26    27    27    NA    NA\n",
      "2.Weather Network                  NA    NA    NA    NA    NA    NA\n",
      "4.Accurate Weather                 28    28    28    27    NA    NA\n",
      "5.Weather Bug                      27    26    27    28    27    27\n",
      "6.BBC Weather                      19    19    19    20    20    NA\n",
      "7.The Weather Channel              NA    NA    NA    NA    NA    NA\n",
      "8.The Skymet Weather               27    26    27    28    27    28\n"
     ]
    }
   ],
   "source": [
    "header = (\"Sites\", \"Current Weather\", \"Day_1\", \"Day_2\", \"Day_3\", \"Day_4\", \"Day_5\")\n",
    "dataset = [\n",
    "    yahoo_data,\n",
    "    msn_weather_data,\n",
    "    weather_network_data,\n",
    "    accurate_weather_data,\n",
    "    weather_bug_data,\n",
    "    bbc_weather_data,\n",
    "    weather_channel_data,\n",
    "    skymet_weather_data,\n",
    "]\n",
    "df = pd.DataFrame(data=dataset, columns=header)\n",
    "print search_text.title()+\"\\n\"\n",
    "print df.set_index(\"Sites\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}