-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtests.py
111 lines (105 loc) · 4.23 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import pandas as pd
import yfinance as yf
import datetime
from joblib import Parallel, delayed
import multiprocessing
import json
import json.decoder
import os
import requests
# tmp = pd.read_csv("nasdaq.csv",sep=",")
# countries_to_exclude = ["India","China","South Korea",
# "Hong Kong", "Malaysia", "Taiwan",
# "Indonesia", "Thailand", "Singapore",
# "Argentina", "Brazil", "Venezuela",
# "Israel", "Qatar", "Russia", "Turkey",
# "Australia", "New Zealand"]
# for i in countries_to_exclude:
# tmp = tmp[tmp["Country"] != i]
with open("stocks.txt","r") as infile:
stock_list = infile.read().split("\n")[:-1]
today = datetime.date.today()
last_week = today - datetime.timedelta(days=7)
two_years = today - datetime.timedelta(days=729)
def processInput(ticker,time):
print(ticker)
if os.path.isfile(f"finished_{time}.txt"):
pass
else:
with open(f"finished_{time}.txt","w") as infile:
infile.write("\n")
if os.path.isfile(f"./data/stock_data_{time}/{ticker}_data.csv"):
with open(f"finished_{time}.txt","a") as finished_file:
finished_file.write(f"{ticker}\n")
with open(f"finished_{time}.txt","r") as infile:
if ticker in infile.read().split("\n"):
return
with open(f"error_file.txt","r") as errorfile:
if ticker in errorfile.read().split("\n"):
return
print(f"{ticker}")
if time == "1h":
ticker_download = yf.download(tickers=f"{ticker}", start=f"{two_years}",end=f"{today}",interval=time,auto_adjust=True,threads=True)
elif time == "1m":
ticker_download = yf.download(tickers=f"{ticker}", start=f"{last_week}",end=f"{today}",interval=time,auto_adjust=True,threads=True)
elif time == "1d":
ticker_download = yf.download(tickers=f"{ticker}", start=f"1970-01-01",end=f"{today}",interval=time,auto_adjust=True,threads=True)
ticker_download.to_csv(f"./data/stock_data_{time}/{ticker}_data.csv",sep=";")
with open(f"finished_{time}.txt","a") as finished_file:
finished_file.write(f"{ticker}\n")
# except requests.exceptions.ConnectionError:
# processInput(ticker)
# except json.decoder.JSONDecodeError:
# with open("error_file.txt","a") as f:
# f.write(ticker)
# return
# except ValueError:
# with open("error_file.txt","a") as f:
# f.write(ticker)
# return
# except IndexError:
# with open("error_file.txt","a") as f:
# f.write(ticker)
# return
# except ValueError:
# with open("error_file.txt","a") as f:
# f.write(ticker)
return
# Parallel(n_jobs=12)(delayed(processInput)(i,"1h") for i in stock_list)
Parallel(n_jobs=12)(delayed(processInput)(i,"1m") for i in stock_list)
# Parallel(n_jobs=12)(delayed(processInput)(i,"1d") for i in stock_list)
def check_stocks():
os.system("ls ./data/stock_data_1h/ > stock_hour.txt")
with open("stock_hour.txt", "r") as stockfile:
stocklist = stockfile.read().split("\n")
for stock in stocklist:
if stock != "":
tmp = pd.read_csv(f"./data/stock_data_1h/{stock}",sep=";")
# tmp = tmp.columns
try:
tmp["Time"] = tmp['Unnamed: 0']
tmp = tmp.drop(['Unnamed: 0'],axis=1)
tmp.to_csv(f"./data/stock_data_1h/{stock}",sep=";",index=False)
# print(tmp)
print(stock)
except KeyError:
try:
print(tmp["Date"])
os.remove(f"./data/stock_data_1h/{stock}")
except KeyError:
print(stock)
pass
# os.remove(f"./data/stock_data_1h/{stock}")
# for i in tmp:
# print(str(i)[:10])
# check_stocks()
# os.system("ls ./data/stock_data_1h/ > stock_hour.txt")
# with open("./stock_hour.txt","r") as stock_file:
# tmp = stock_file.read().split("\n")
# stock_list = []
# for i in tmp:
# if i != "":
# stock_list.append(i[:-9])
# with open("stocks.txt","w") as outfile:
# for i in stock_list:
# outfile.write(f"{i}\n")