Bug/#250 stock data 문제 해결

packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts

@@ -31,7 +31,7 @@ export class OpenapiDetailData {
     private readonly datasource: DataSource,
     @Inject('winston') private readonly logger: Logger,
   ) {
-    //setTimeout(() => this.getDetailData(), 5000);
+    //this.getDetailData();
   }
 
   @Cron('0 8 * * 1-5')
@@ -191,7 +191,7 @@ export class OpenapiDetailData {
         dataQuery,
         TR_IDS.FINANCIAL_DATA,
       );
-      if (response.output) {
+      if (response.output && response.output[0]) {
         const output1 = response.output;
         return output1[0];
       }
@@ -224,8 +224,6 @@ export class OpenapiDetailData {
     const output1 = await this.getFinancialRatio(stock, conf);
     const output2 = await this.getProductData(stock, conf);
 
-    this.logger.info(JSON.stringify(output1));
-    this.logger.info(JSON.stringify(output2));
     if (isFinancialRatioData(output1) && isProductDetail(output2)) {
       const stockDetail = await this.makeStockDetailObject(
         output1,
@@ -236,7 +234,7 @@ export class OpenapiDetailData {
       const kospiStock = await this.makeKospiStockObject(output2, stock.id!);
       this.saveKospiData(kospiStock);
 
-      this.logger.info(`${stock.id!} is saved`);
+      this.logger.info(`${stock.id!} detail data is saved`);
     }
   }
 

packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts

@@ -1,6 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { Cron } from '@nestjs/schedule';
-import { DataSource, EntityManager } from 'typeorm';
+import { DataSource } from 'typeorm';
 import { Logger } from 'winston';
 import {
   ChartData,
@@ -32,13 +32,13 @@ const DATE_TO_ENTITY = {
 };
 
 const DATE_TO_MONTH = {
-  D: 3,
+  D: 1,
   W: 6,
-  M: 12,
-  Y: 24,
+  M: 24,
+  Y: 120,
 };
 
-const INTERVALS = 4000;
+const INTERVALS = 10000;
 
 @Injectable()
 export class OpenapiPeriodData {
@@ -48,9 +48,7 @@ export class OpenapiPeriodData {
     private readonly datasource: DataSource,
     private readonly openApiToken: OpenapiTokenApi,
     @Inject('winston') private readonly logger: Logger,
-  ) {
-    //this.getItemChartPriceCheck();
-  }
+  ) {}
 
   @Cron('0 1 * * 1-5')
   async getItemChartPriceCheck() {
@@ -60,20 +58,15 @@ export class OpenapiPeriodData {
         isTrading: true,
       },
     });
-    const configCount = this.openApiToken.configs.length;
-    const chunkSize = Math.ceil(stocks.length / configCount);
-
-    for (let i = 0; i < configCount; i++) {
-      const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize);
-      this.getChartData(chunk, 'D');
-      setTimeout(() => this.getChartData(chunk, 'W'), INTERVALS);
-      setTimeout(() => this.getChartData(chunk, 'M'), INTERVALS * 2);
-      setTimeout(() => this.getChartData(chunk, 'Y'), INTERVALS * 3);
-    }
+
+    await this.getChartData(stocks, 'Y');
+    await this.getChartData(stocks, 'M');
+    await this.getChartData(stocks, 'W');
+    await this.getChartData(stocks, 'D');
   }
 
   private async getChartData(chunk: Stock[], period: Period) {
-    const baseTime = INTERVALS * 4;
+    const baseTime = INTERVALS;
     const entity = DATE_TO_ENTITY[period];
 
     let time = 0;
@@ -89,26 +82,23 @@ export class OpenapiPeriodData {
     entity: typeof StockData,
   ) {
     const stockPeriod = new StockData();
-    const manager = this.datasource.manager;
     let configIdx = 0;
     let end = getTodayDate();
-    let start = getPreviousDate(end, 3);
+    let start = getPreviousDate(end, DATE_TO_MONTH[period]);
     let isFail = false;
 
     while (!isFail) {
+      await new Promise((resolve) => setTimeout(resolve, INTERVALS / 10));
       configIdx = (configIdx + 1) % (await this.openApiToken.configs()).length;
       this.setStockPeriod(stockPeriod, stock.id!, end);
 
-      // chart 데이터가 있는 지 확인 -> 리턴
-      if (await this.existsChartData(stockPeriod, manager, entity)) return;
-
       const query = this.getItemChartPriceQuery(stock.id!, start, end, period);
 
       const output = await this.fetchChartData(query, configIdx);
 
       if (output) {
         await this.saveChartData(entity, stock.id!, output);
-        ({ endDate: end, startDate: start } = this.updateDates(start, period));
+        ({ endDate: end, startDate: start } = this.updateDates(end, period));
       } else isFail = true;
     }
   }
@@ -137,34 +127,32 @@ export class OpenapiPeriodData {
       return response.output2 as ChartData[];
     } catch (error) {
       this.logger.warn(error);
+      setTimeout(() => this.fetchChartData(query, configIdx), INTERVALS / 10);
     }
   }
 
   private updateDates(
-    startDate: string,
+    endDate: string,
     period: Period,
   ): { endDate: string; startDate: string } {
-    const endDate = getPreviousDate(startDate, DATE_TO_MONTH[period]);
-    startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]);
+    endDate = getPreviousDate(endDate, DATE_TO_MONTH[period]);
+    const startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]);
     return { endDate, startDate };
   }
 
-  private async existsChartData(
-    stock: StockData,
-    manager: EntityManager,
-    entity: typeof StockData,
-  ) {
+  private async existsChartData(stock: StockData, entity: typeof StockData) {
+    const manager = this.datasource.manager;
     return await manager.findOne(entity, {
       where: {
         stock: { id: stock.stock.id },
-        createdAt: stock.startTime,
+        startTime: stock.startTime,
       },
     });
   }
 
   private async insertChartData(stock: StockData, entity: typeof StockData) {
     const manager = this.datasource.manager;
-    if (!(await this.existsChartData(stock, manager, entity))) {
+    if (!(await this.existsChartData(stock, entity))) {
       await manager.save(entity, stock);
     }
   }

packages/backend/src/scraper/openapi/api/openapiToken.api.ts

@@ -42,6 +42,13 @@ export class OpenapiTokenApi {
 
   @Cron('30 0 * * 1-5')
   async init() {
+    const expired_config = this.config.filter(
+      (val) =>
+        this.isTokenExpired(val.STOCK_API_TIMEOUT) &&
+        this.isTokenExpired(val.STOCK_WEBSOCKET_TIMEOUT),
+    );
+    const isUndefined = this.config[0].STOCK_WEBSOCKET_TIMEOUT ? false : true;
+    if (!isUndefined && !expired_config.length) return;
     const tokens = this.convertConfigToTokenEntity(this.config);
     const config = await this.getPropertyFromDB(tokens);
     const expired = config.filter(
@@ -79,6 +86,8 @@ export class OpenapiTokenApi {
         STOCK_API_TOKEN: val.api_token,
         STOCK_URL: val.api_url,
         STOCK_WEBSOCKET_KEY: val.websocket_key,
+        STOCK_API_TIMEOUT: val.api_token_expire,
+        STOCK_WEBSOCKET_TIMEOUT: val.websocket_key_expire,
       };
       result.push(config);
     });

packages/backend/src/scraper/openapi/config/openapi.config.ts

@@ -9,6 +9,8 @@ export const openApiConfig: {
   STOCK_API_PASSWORD: string | undefined;
   STOCK_API_TOKEN?: string;
   STOCK_WEBSOCKET_KEY?: string;
+  STOCK_API_TIMEOUT?: Date;
+  STOCK_WEBSOCKET_TIMEOUT?: Date;
 } = {
   STOCK_URL: process.env.STOCK_URL,
   STOCK_ACCOUNT: process.env.STOCK_ACCOUNT,

packages/backend/src/scraper/openapi/liveData.service.ts

@@ -24,13 +24,11 @@ export class LiveData {
     @Inject('winston') private readonly logger: Logger,
   ) {
     this.connect();
-    this.subscribe('000020');
   }
 
   private async openapiSubscribe(stockId: string) {
     const config = (await this.openApiToken.configs())[0];
     const result = await this.openapiLiveData.connectLiveData(stockId, config);
-    this.logger.info(JSON.stringify(result));
     try {
       const stockLiveData = this.openapiLiveData.convertResponseToStockLiveData(
         result.output,
@@ -58,6 +56,7 @@ export class LiveData {
       this.webSocketClient.subscribe(message);
     }
   }
+
 
   async discribe(stockId: string) {
     if (this.clientStock.has(stockId)) {
@@ -90,13 +89,10 @@ export class LiveData {
         const message = this.parseMessage(data);
         if (message.header) {
           if (message.header.tr_id === 'PINGPONG') {
-            this.logger.info(`Received PING: ${data}`);
             client.pong(data);
           }
           return;
         }
-        this.logger.info(`Recived data : ${data}`);
-        this.logger.info(`Stock id : ${message[0]['STOCK_ID']}`);
         const liveData = this.openapiLiveData.convertLiveData(message);
         await this.openapiLiveData.saveLiveData(liveData);
       } catch (error) {
@@ -142,7 +138,6 @@ export class LiveData {
     stockId: string,
     tr_type: TR_IDS,
   ): string {
-    this.logger.info(JSON.stringify(config));
     const message = {
       header: {
         approval_key: config.STOCK_WEBSOCKET_KEY!,

packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts

@@ -106,6 +106,7 @@ export type ProductDetail = {
 
 export const isProductDetail = (data: any): data is ProductDetail => {
   return (
+    data &&
     typeof data.pdno === 'string' &&
     typeof data.prdt_type_cd === 'string' &&
     typeof data.mket_id_cd === 'string' &&

packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts

@@ -12,25 +12,13 @@ export class WebsocketClient {
   constructor(@Inject('winston') private readonly logger: Logger) {}
 
   subscribe(message: string) {
-    this.logger.info(`Subscribe : ${message}`);
     this.sendMessage(message);
   }
 
   discribe(message: string) {
-    this.logger.info(`Discribe : ${message}`);
     this.sendMessage(message);
   }
 
-  // TODO : 분리
-  private initDisconnect(
-    initCloseCallback: () => void,
-    initErrorCallback: (error: unknown) => void,
-  ) {
-    this.client.on('close', initCloseCallback);
-
-    this.client.on('error', initErrorCallback);
-  }
-
   private initOpen(fn: () => void) {
     this.client.on('open', fn);
   }
@@ -39,6 +27,14 @@ export class WebsocketClient {
     this.client.on('message', fn);
   }
 
+  private initDisconnect(initCloseCallback: () => void) {
+    this.client.on('close', initCloseCallback);
+  }
+
+  private initError(initErrorCallback: (error: unknown) => void) {
+    this.client.on('error', initErrorCallback);
+  }
+
   connectPacade(
     initOpenCallback: (fn: (message: string) => void) => () => void,
     initMessageCallback: (client: WebSocket) => (data: RawData) => void,
@@ -47,7 +43,8 @@ export class WebsocketClient {
   ) {
     this.initOpen(initOpenCallback(this.sendMessage));
     this.initMessage(initMessageCallback(this.client));
-    this.initDisconnect(initCloseCallback, initErrorCallback);
+    this.initDisconnect(initCloseCallback);
+    this.initError(initErrorCallback);
   }
 
   private sendMessage(message: string) {

packages/backend/src/stock/domain/stockData.entity.ts

@@ -5,9 +5,11 @@ import {
   CreateDateColumn,
   JoinColumn,
   ManyToOne,
+  Index,
 } from 'typeorm';
 import { Stock } from './stock.entity';
 
+@Index('stock_id_start_time', ['stock.id', 'startTime'], { unique: true })
 export class StockData {
   @PrimaryGeneratedColumn()
   id: number;