From 8b33b3f11a0f8c30f454541fa9f5c18ef33726d1 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 14 Nov 2024 11:46:39 +0900 Subject: [PATCH 001/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20swagge?= =?UTF-8?q?r=20url=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/swagger.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/configs/swagger.config.ts b/packages/backend/src/configs/swagger.config.ts index c9d9dfd5..ad58834f 100644 --- a/packages/backend/src/configs/swagger.config.ts +++ b/packages/backend/src/configs/swagger.config.ts @@ -9,5 +9,5 @@ export function useSwagger(app: INestApplication) { .build(); const documentFactory = () => SwaggerModule.createDocument(app, config); - SwaggerModule.setup('api', app, documentFactory); + SwaggerModule.setup('swagger', app, documentFactory); } From 6f19e72e93076061adf2e5d067bf61f1037795ca Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 16 Nov 2024 20:44:06 +0900 Subject: [PATCH 002/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=9D=B4=EC=A0=84?= =?UTF-8?q?=20=EC=B1=84=ED=8C=85=20=EC=8A=A4=ED=81=AC=EB=A1=A4=20=EC=A1=B0?= =?UTF-8?q?=ED=9A=8C=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 39 +++++++++++ packages/backend/src/chat/chat.gateway.ts | 19 +++-- packages/backend/src/chat/chat.module.ts | 4 +- packages/backend/src/chat/chat.service.ts | 69 +++++++++++++++++-- .../backend/src/chat/domain/chat.entity.ts | 2 +- packages/backend/src/chat/dto/chat.request.ts | 30 ++++++++ .../backend/src/chat/dto/chat.response.ts | 44 ++++++++++++ .../backend/src/common/dateEmbedded.entity.ts | 4 +- 8 files changed, 195 insertions(+), 16 deletions(-) create mode 100644 packages/backend/src/chat/chat.controller.ts create mode 100644 packages/backend/src/chat/dto/chat.request.ts create mode 100644 packages/backend/src/chat/dto/chat.response.ts diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts new file mode 100644 index 00000000..f2a5452d --- /dev/null +++ b/packages/backend/src/chat/chat.controller.ts @@ -0,0 +1,39 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { + ApiBadRequestResponse, + ApiOkResponse, + ApiOperation, +} from '@nestjs/swagger'; +import { ChatService } from '@/chat/chat.service'; +import { ChatScrollRequest } from '@/chat/dto/chat.request'; +import { ChatScrollResponse } from '@/chat/dto/chat.response'; + +@Controller('chat') +export class ChatController { + constructor(private readonly chatService: ChatService) {} + + @ApiOperation({ + summary: '채팅 스크롤 조회 API', + description: '채팅을 스크롤하여 조회한다.', + }) + @ApiOkResponse({ + description: '스크롤 조회 성공', + type: ChatScrollResponse, + }) + @ApiBadRequestResponse({ + description: '스크롤 크기 100 초과', + example: { + message: 'pageSize should be less than 100', + error: 'Bad Request', + statusCode: 400, + }, + }) + @Get() + async findChatList(@Query() request: ChatScrollRequest) { + return await this.chatService.scrollNextChat( + request.stockId, + request.latestChatId, + request.pageSize, + ); + } +} diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index 322f35e9..f5fc099e 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -67,18 +67,23 @@ export class ChatGateway implements OnGatewayConnection { async handleConnection(client: Socket) { const room = client.handshake.query.stockId; - if (!room || !(await this.stockService.checkStockExist(room as string))) { + if ( + !this.isString(room) || + !(await this.stockService.checkStockExist(room)) + ) { client.emit('error', 'Invalid stockId'); this.logger.warn(`client connected with invalid stockId: ${room}`); client.disconnect(); return; } - if (room) { - client.join(room); - const messages = await this.chatService.getChatList(room as string); - this.logger.info(`client joined room ${room}`); - client.emit('chat', messages); - } + client.join(room); + const messages = await this.chatService.scrollFirstChat(room); + this.logger.info(`client joined room ${room}`); + client.emit('chat', messages); + } + + private isString(value: string | string[] | undefined): value is string { + return typeof value === 'string'; } private toResponse(chat: Chat): chatResponse { diff --git a/packages/backend/src/chat/chat.module.ts b/packages/backend/src/chat/chat.module.ts index 345a0f76..089b37e4 100644 --- a/packages/backend/src/chat/chat.module.ts +++ b/packages/backend/src/chat/chat.module.ts @@ -1,13 +1,15 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { SessionModule } from '@/auth/session.module'; +import { ChatController } from '@/chat/chat.controller'; import { ChatGateway } from '@/chat/chat.gateway'; +import { ChatService } from '@/chat/chat.service'; import { Chat } from '@/chat/domain/chat.entity'; import { StockModule } from '@/stock/stock.module'; -import { ChatService } from '@/chat/chat.service'; @Module({ imports: [TypeOrmModule.forFeature([Chat]), StockModule, SessionModule], + controllers: [ChatController], providers: [ChatGateway, ChatService], }) export class ChatModule {} diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 624618bd..bfdf5c28 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -1,12 +1,15 @@ import { Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; +import { ChatScrollResponse } from '@/chat/dto/chat.response'; -interface ChatMessage { +export interface ChatMessage { message: string; stockId: string; } +const DEFAULT_PAGE_SIZE = 20; + @Injectable() export class ChatService { constructor(private readonly dataSource: DataSource) {} @@ -19,13 +22,69 @@ export class ChatService { }); } - async getChatList(stockId: string) { + async scrollFirstChat(stockId: string, scrollSize?: number) { + const result = await this.findFirstChatScroll(stockId, scrollSize); + return await this.toScrollResponse(result, scrollSize); + } + + async scrollNextChat( + stockId: string, + latestChatId?: number, + pageSize?: number, + ) { + const result = await this.findChatScroll(stockId, latestChatId, pageSize); + return await this.toScrollResponse(result, pageSize); + } + + private async toScrollResponse(result: Chat[], pageSize: number | undefined) { + const hasMore = + !!result && result.length > (pageSize ? pageSize : DEFAULT_PAGE_SIZE); + if (hasMore) { + result.pop(); + } + return new ChatScrollResponse(result, hasMore); + } + + private async findChatScroll( + stockId: string, + latestChatId?: number, + pageSize?: number, + ) { + if (!latestChatId) { + return await this.findFirstChatScroll(stockId, pageSize); + } else { + return await this.findNextChatScroll(stockId, latestChatId, pageSize); + } + } + + private async findFirstChatScroll(stockId: string, pageSize?: number) { const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - console.log(stockId); + if (!pageSize) { + pageSize = DEFAULT_PAGE_SIZE; + } return queryBuilder .where('chat.stock_id = :stockId', { stockId }) - .orderBy('chat.created_at', 'DESC') - .limit(100) + .orderBy('chat.id', 'DESC') + .limit(pageSize + 1) + .getMany(); + } + + private async findNextChatScroll( + stockId: string, + latestChatId: number, + pageSize?: number, + ) { + const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); + if (!pageSize) { + pageSize = DEFAULT_PAGE_SIZE; + } + return queryBuilder + .where('chat.stock_id = :stockId and chat.id < :latestChatId', { + stockId, + latestChatId, + }) + .orderBy('chat.id', 'DESC') + .limit(pageSize + 1) .getMany(); } } diff --git a/packages/backend/src/chat/domain/chat.entity.ts b/packages/backend/src/chat/domain/chat.entity.ts index 9406a2e4..13800bff 100644 --- a/packages/backend/src/chat/domain/chat.entity.ts +++ b/packages/backend/src/chat/domain/chat.entity.ts @@ -33,5 +33,5 @@ export class Chat { likeCount: number = 0; @Column(() => DateEmbedded, { prefix: '' }) - date?: DateEmbedded; + date: DateEmbedded; } diff --git a/packages/backend/src/chat/dto/chat.request.ts b/packages/backend/src/chat/dto/chat.request.ts new file mode 100644 index 00000000..f3260509 --- /dev/null +++ b/packages/backend/src/chat/dto/chat.request.ts @@ -0,0 +1,30 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNumber, IsOptional, IsString } from 'class-validator'; + +export class ChatScrollRequest { + @ApiProperty({ + description: '종목 주식 id(종목방 id)', + example: 'A005930', + }) + @IsString() + readonly stockId: string; + + @ApiProperty({ + description: '최신 채팅 id', + example: 99999, + required: false, + }) + @IsOptional() + @IsNumber() + readonly latestChatId?: number; + + @ApiProperty({ + description: '페이지 크기', + example: 20, + default: 20, + required: false, + }) + @IsOptional() + @IsNumber() + readonly pageSize?: number; +} diff --git a/packages/backend/src/chat/dto/chat.response.ts b/packages/backend/src/chat/dto/chat.response.ts new file mode 100644 index 00000000..48aacb0d --- /dev/null +++ b/packages/backend/src/chat/dto/chat.response.ts @@ -0,0 +1,44 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Chat } from '@/chat/domain/chat.entity'; +import { ChatType } from '@/chat/domain/chatType.enum'; + +interface ChatResponse { + id: number; + likeCount: number; + message: string; + type: string; + createdAt: Date; +} + +export class ChatScrollResponse { + @ApiProperty({ + description: '다음 페이지가 있는지 여부', + example: true, + }) + readonly hasMore: boolean; + + @ApiProperty({ + description: '채팅 목록', + example: [ + { + id: 1, + likeCount: 0, + message: '안녕하세요', + type: ChatType.NORMAL, + createdAt: new Date(), + }, + ], + }) + readonly chats: ChatResponse[]; + + constructor(chats: Chat[], hasMore: boolean) { + this.chats = chats.map((chat) => ({ + id: chat.id, + likeCount: chat.likeCount, + message: chat.message, + type: chat.type, + createdAt: chat.date!.createdAt, + })); + this.hasMore = hasMore; + } +} diff --git a/packages/backend/src/common/dateEmbedded.entity.ts b/packages/backend/src/common/dateEmbedded.entity.ts index 2a0c9dd8..a16c1cf9 100644 --- a/packages/backend/src/common/dateEmbedded.entity.ts +++ b/packages/backend/src/common/dateEmbedded.entity.ts @@ -2,8 +2,8 @@ import { CreateDateColumn, UpdateDateColumn } from 'typeorm'; export class DateEmbedded { @CreateDateColumn({ type: 'timestamp', name: 'created_at' }) - createdAt?: Date; + createdAt: Date; @UpdateDateColumn({ type: 'timestamp', name: 'updated_at' }) - updatedAt?: Date; + updatedAt: Date; } From 8c2796edc5bb29f4a153fc67b64aa527e39f77d8 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 16 Nov 2024 20:44:31 +0900 Subject: [PATCH 003/112] =?UTF-8?q?=E2=9C=A8=20feat:=20class=20dto=20?= =?UTF-8?q?=EB=82=B4=EB=B6=80=20=ED=83=80=EC=9E=85=20=EC=9E=90=EB=8F=99=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/main.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 311bcdb9..73012397 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -11,7 +11,12 @@ async function bootstrap() { const app = await NestFactory.create(AppModule); const store = app.get(MEMORY_STORE); app.use(session({ ...sessionConfig, store })); - app.useGlobalPipes(new ValidationPipe({ transform: true })); + app.useGlobalPipes( + new ValidationPipe({ + transform: true, + transformOptions: { enableImplicitConversion: true }, + }), + ); useSwagger(app); app.use(passport.initialize()); app.use(passport.session()); From f6369d8e4fc8d84b0952800375cf8a45c0254652 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sun, 17 Nov 2024 20:33:07 +0900 Subject: [PATCH 004/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A1=A4=20=ED=81=AC=EA=B8=B0=EB=A5=BC=20100=EC=9D=84=20?= =?UTF-8?q?=EB=84=98=EC=A7=80=20=EC=95=8A=EB=8F=84=EB=A1=9D=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.service.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index bfdf5c28..ef262e7b 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; @@ -23,6 +23,7 @@ export class ChatService { } async scrollFirstChat(stockId: string, scrollSize?: number) { + this.validatePageSize(scrollSize); const result = await this.findFirstChatScroll(stockId, scrollSize); return await this.toScrollResponse(result, scrollSize); } @@ -32,10 +33,17 @@ export class ChatService { latestChatId?: number, pageSize?: number, ) { + this.validatePageSize(pageSize); const result = await this.findChatScroll(stockId, latestChatId, pageSize); return await this.toScrollResponse(result, pageSize); } + private validatePageSize(scrollSize?: number) { + if (scrollSize && scrollSize > 100) { + throw new BadRequestException('pageSize should be less than 100'); + } + } + private async toScrollResponse(result: Chat[], pageSize: number | undefined) { const hasMore = !!result && result.length > (pageSize ? pageSize : DEFAULT_PAGE_SIZE); From e2e3938de1a0e4a45744b7cd9ef6b2483450d3fd Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sun, 17 Nov 2024 20:41:22 +0900 Subject: [PATCH 005/112] =?UTF-8?q?=E2=9C=85=20test:=20100=EA=B0=9C=20?= =?UTF-8?q?=EC=B4=88=EA=B3=BC=20=EB=A9=94=EC=8B=9C=EC=A7=80=20=EC=A1=B0?= =?UTF-8?q?=ED=9A=8C=EC=97=90=20=EB=8C=80=ED=95=9C=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8=20=EC=BD=94=EB=93=9C=20=EC=9E=91=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/chat/chat.service.spec.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/backend/src/chat/chat.service.spec.ts diff --git a/packages/backend/src/chat/chat.service.spec.ts b/packages/backend/src/chat/chat.service.spec.ts new file mode 100644 index 00000000..7d181061 --- /dev/null +++ b/packages/backend/src/chat/chat.service.spec.ts @@ -0,0 +1,23 @@ +import { DataSource } from 'typeorm'; +import { ChatService } from '@/chat/chat.service'; +import { createDataSourceMock } from '@/user/user.service.spec'; + +describe('ChatService 테스트', () => { + test('첫 스크롤을 조회시 100개 이상 조회하면 예외가 발생한다.', async () => { + const dataSource = createDataSourceMock({}); + const chatService = new ChatService(dataSource as DataSource); + + await expect(() => + chatService.scrollNextChat('A005930', 1, 101), + ).rejects.toThrow('pageSize should be less than 100'); + }); + + test('100개 이상의 채팅을 조회하려 하면 예외가 발생한다.', async () => { + const dataSource = createDataSourceMock({}); + const chatService = new ChatService(dataSource as DataSource); + + await expect(() => + chatService.scrollFirstChat('A005930', 101), + ).rejects.toThrow('pageSize should be less than 100'); + }); +}); From 0bb7c83848259d3f8768af0f5ec6c0990f106f35 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 18 Nov 2024 16:46:04 +0900 Subject: [PATCH 006/112] =?UTF-8?q?=E2=9C=A8=20feat:=20detail=20=ED=95=AD?= =?UTF-8?q?=EB=AA=A9=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20=EA=B5=AC=ED=98=84?= =?UTF-8?q?=20=EC=A7=84=ED=96=89=EC=A4=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 102 +++++++++ .../openapi/api/openapiPeriodData.api.ts | 8 +- .../src/scraper/openapi/openapiUtil.api.ts | 5 +- .../openapi/type/openapiDetailData.type.ts | 213 ++++++++++++++++++ .../scraper/openapi/type/openapiUtil.type.ts | 7 + .../src/stock/domain/kospiStock.entity.ts | 0 .../backend/src/stock/domain/stock.entity.ts | 4 +- .../src/stock/domain/stockData.entity.ts | 4 +- 8 files changed, 334 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiUtil.type.ts create mode 100644 packages/backend/src/stock/domain/kospiStock.entity.ts diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts new file mode 100644 index 00000000..643f7636 --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -0,0 +1,102 @@ +import { Cron } from '@nestjs/schedule'; +import { DataSource } from 'typeorm'; +import { openApiConfig } from '../config/openapi.config'; +import { getOpenApi } from '../openapiUtil.api'; +import { + DetailDataQuery, + FinancialData, + FinancialDetail, + isFinancialData, + isFinancialDetail, + isProductDetail, + ProductDetail, + StockDetailQuery, +} from '../type/openapiDetailData.type'; +import { openApiToken } from './openapiToken.api'; +import { Stock } from '@/stock/domain/stock.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; +import { StockDaily } from '@/stock/domain/stockData.entity'; + +export class OpenapiDetailData { + private readonly financialUrl: string = + '/uapi/domestic-stock/v1/finance/financial-ratio'; + private readonly defaultUrl: string = + '/uapi/domestic-stock/v1/quotations/search-stock-info'; + private readonly incomeUrl: string = '/uapi/domestic-stock/v1/finance/income-statement'; + private readonly intervals = 4000; + private readonly config: (typeof openApiConfig)[] = openApiToken.configs; + constructor(private readonly datasource: DataSource) {} + + @Cron('10 1 * * 1-5') + public async getDetailData() { + const entityManager = this.datasource.manager; + const stocks = await entityManager.find(Stock); + const configCount = this.config.length; + + const chunkSize = Math.ceil(stocks.length / configCount); + + for (let i = 0; i < configCount; i++) { + const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); + this.getDetailDataChunk(chunk, this.config[i]); + } + } + + private async saveDetailData(output1: FinancialData, output2: ProductDetail, output3 : StockDaily[]) { + const entityManager = this.datasource.manager; + const entity = StockDetail; + entityManager.create(entity, output1); + } + + private makeStockDetailObject( + output1: FinancialDetail, + output2: ProductDetail, + ): StockDetail { + const result = new StockDetail(); + result.marketCap = output2. + return result; + } + + private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { + const manager = this.datasource.manager; + for (const stock of chunk) { + const dataQuery = this.getDetailDataQuery(stock.id!); + const defaultQuery = this.getDefaultDataQuery(stock.id!); + const output1 = await getOpenApi(this.incomeUrl, conf, dataQuery, 'FHKST66430200'); + const output2 = await getOpenApi( + this.defaultUrl, + conf, + defaultQuery, + 'CTPF1002R', + ); + const output3 = await manager.find(StockDaily, { + where: { + + } + }) + if (isFinancialDetail(output1) && isProductDetail(output2)) { + } + } + } + + private getDefaultDataQuery( + stockId: string, + code: '300' | '301' | '302' | '306' = '300', + ): StockDetailQuery { + return { + pdno: stockId, + code: code, + }; + } + + private getDetailDataQuery( + stockId: string, + divCode: 'J' = 'J', + classify: '0' | '1' = '1', + ): DetailDataQuery { + return { + fid_cond_mrkt_div_code: divCode, + fid_input_iscd: stockId, + fid_div_cls_code: classify, + }; + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index a11766b6..f06efcb0 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -36,13 +36,13 @@ const INTERVALS = 4000; export class OpenapiPeriodData { private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; - public constructor(private readonly datasourse: DataSource) { + public constructor(private readonly datasource: DataSource) { //this.getItemChartPriceCheck(); } @Cron('0 1 * * 1-5') public async getItemChartPriceCheck() { - const entityManager = this.datasourse.manager; + const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); const configCount = openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); @@ -59,7 +59,7 @@ export class OpenapiPeriodData { private async getChartData(chunk: Stock[], period: Period) { const baseTime = INTERVALS * 4; const entity = DATE_TO_ENTITY[period]; - const manager = this.datasourse.manager; + const manager = this.datasource.manager; let time = 0; for (const stock of chunk) { @@ -148,7 +148,7 @@ export class OpenapiPeriodData { } private async insertChartData(stock: StockData, entity: typeof StockData) { - const manager = this.datasourse.manager; + const manager = this.datasource.manager; if (!(await this.existsChartData(stock, manager, entity))) { await manager.save(entity, stock); } diff --git a/packages/backend/src/scraper/openapi/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/openapiUtil.api.ts index e09c181c..06ae8c23 100644 --- a/packages/backend/src/scraper/openapi/openapiUtil.api.ts +++ b/packages/backend/src/scraper/openapi/openapiUtil.api.ts @@ -1,5 +1,6 @@ import axios from 'axios'; import { openApiConfig } from './config/openapi.config'; +import { DEFAULT_TR_ID, TR_ID } from './type/openapiUtil.type'; const postOpenApi = async ( url: string, @@ -18,6 +19,7 @@ const getOpenApi = async ( url: string, config: typeof openApiConfig, query: object, + tr_id: TR_ID = DEFAULT_TR_ID, ) => { try { const response = await axios.get(config.STOCK_URL + url, { @@ -26,7 +28,8 @@ const getOpenApi = async ( Authorization: `Bearer ${config.STOCK_API_TOKEN}`, appkey: config.STOCK_API_KEY, appsecret: config.STOCK_API_PASSWORD, - tr_id: 'FHKST03010100', + tr_id, + custtype: 'P', }, }); return response.data; diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts new file mode 100644 index 00000000..a4267b40 --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -0,0 +1,213 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +/* eslint-disable max-lines-per-function */ + +export type DetailDataQuery = { + fid_cond_mrkt_div_code: 'J'; + fid_input_iscd: string; + fid_div_cls_code: '0' | '1'; +}; +export type FinancialData = { + stac_yymm: string; // 결산 년월 + grs: string; // 매출액 증가율 + bsop_prfi_inrt: string; // 영업 이익 증가율 + ntin_inrt: string; // 순이익 증가율 + roe_val: string; // ROE 값 + eps: string; // EPS + sps: string; // 주당매출액 + bps: string; // BPS + rsrv_rate: string; // 유보 비율 + lblt_rate: string; // 부채 비율 +}; + +export function isFinancialData(data: any): data is FinancialData { + return ( + data && + typeof data.stac_yymm === 'string' && + typeof data.grs === 'string' && + typeof data.bsop_prfi_inrt === 'string' && + typeof data.ntin_inrt === 'string' && + typeof data.roe_val === 'string' && + typeof data.eps === 'string' && + typeof data.sps === 'string' && + typeof data.bps === 'string' && + typeof data.rsrv_rate === 'string' && + typeof data.lblt_rate === 'string' + ); +} + +export type ProductDetail = { + pdno: string; // 상품번호 + prdt_type_cd: string; // 상품유형코드 + mket_id_cd: string; // 시장ID코드 + scty_grp_id_cd: string; // 증권그룹ID코드 + excg_dvsn_cd: string; // 거래소구분코드 + setl_mmdd: string; // 결산월일 + lstg_stqt: string; // 상장주수 + lstg_cptl_amt: string; // 상장자본금액 + cpta: string; // 자본금 + papr: string; // 액면가 + issu_pric: string; // 발행가격 + kospi200_item_yn: string; // 코스피200종목여부 + scts_mket_lstg_dt: string; // 유가증권시장상장일자 + scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 + kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 + kosdaq_mket_lstg_abol_dt: string; // 코스닥시장상장폐지일자 + frbd_mket_lstg_dt: string; // 프리보드시장상장일자 + frbd_mket_lstg_abol_dt: string; // 프리보드시장상장폐지일자 + reits_kind_cd: string; // 리츠종류코드 + etf_dvsn_cd: string; // ETF구분코드 + oilf_fund_yn: string; // 유전펀드여부 + idx_bztp_lcls_cd: string; // 지수업종대분류코드 + idx_bztp_mcls_cd: string; // 지수업종중분류코드 + idx_bztp_scls_cd: string; // 지수업종소분류코드 + stck_kind_cd: string; // 주식종류코드 + mfnd_opng_dt: string; // 뮤추얼펀드개시일자 + mfnd_end_dt: string; // 뮤추얼펀드종료일자 + dpsi_erlm_cncl_dt: string; // 예탁등록취소일자 + etf_cu_qty: string; // ETFCU수량 + prdt_name: string; // 상품명 + prdt_name120: string; // 상품명120 + prdt_abrv_name: string; // 상품약어명 + std_pdno: string; // 표준상품번호 + prdt_eng_name: string; // 상품영문명 + prdt_eng_name120: string; // 상품영문명120 + prdt_eng_abrv_name: string; // 상품영문약어명 + dpsi_aptm_erlm_yn: string; // 예탁지정등록여부 + etf_txtn_type_cd: string; // ETF과세유형코드 + etf_type_cd: string; // ETF유형코드 + lstg_abol_dt: string; // 상장폐지일자 + nwst_odst_dvsn_cd: string; // 신주구주구분코드 + sbst_pric: string; // 대용가격 + thco_sbst_pric: string; // 당사대용가격 + thco_sbst_pric_chng_dt: string; // 당사대용가격변경일자 + tr_stop_yn: string; // 거래정지여부 + admn_item_yn: string; // 관리종목여부 + thdt_clpr: string; // 당일종가 + bfdy_clpr: string; // 전일종가 + clpr_chng_dt: string; // 종가변경일자 + std_idst_clsf_cd: string; // 표준산업분류코드 + std_idst_clsf_cd_name: string; // 표준산업분류코드명 + idx_bztp_lcls_cd_name: string; // 지수업종대분류코드명 + idx_bztp_mcls_cd_name: string; // 지수업종중분류코드명 + idx_bztp_scls_cd_name: string; // 지수업종소분류코드명 + ocr_no: string; // OCR번호 + crfd_item_yn: string; // 크라우드펀딩종목여부 + elec_scty_yn: string; // 전자증권여부 + issu_istt_cd: string; // 발행기관코드 + etf_chas_erng_rt_dbnb: string; // ETF추적수익율배수 + etf_etn_ivst_heed_item_yn: string; // ETFETN투자유의종목여부 + stln_int_rt_dvsn_cd: string; // 대주이자율구분코드 + frnr_psnl_lmt_rt: string; // 외국인개인한도비율 + lstg_rqsr_issu_istt_cd: string; // 상장신청인발행기관코드 + lstg_rqsr_item_cd: string; // 상장신청인종목코드 + trst_istt_issu_istt_cd: string; // 신탁기관발행기관코드 +}; + +export const isProductDetail = (data: any): data is ProductDetail => { + return ( + typeof data.pdno === 'string' && + typeof data.prdt_type_cd === 'string' && + typeof data.mket_id_cd === 'string' && + typeof data.scty_grp_id_cd === 'string' && + typeof data.excg_dvsn_cd === 'string' && + typeof data.setl_mmdd === 'string' && + typeof data.lstg_stqt === 'string' && + typeof data.lstg_cptl_amt === 'string' && + typeof data.cpta === 'string' && + typeof data.papr === 'string' && + typeof data.issu_pric === 'string' && + typeof data.kospi200_item_yn === 'string' && + typeof data.scts_mket_lstg_dt === 'string' && + typeof data.scts_mket_lstg_abol_dt === 'string' && + typeof data.kosdaq_mket_lstg_dt === 'string' && + typeof data.kosdaq_mket_lstg_abol_dt === 'string' && + typeof data.frbd_mket_lstg_dt === 'string' && + typeof data.frbd_mket_lstg_abol_dt === 'string' && + typeof data.reits_kind_cd === 'string' && + typeof data.etf_dvsn_cd === 'string' && + typeof data.oilf_fund_yn === 'string' && + typeof data.idx_bztp_lcls_cd === 'string' && + typeof data.idx_bztp_mcls_cd === 'string' && + typeof data.idx_bztp_scls_cd === 'string' && + typeof data.stck_kind_cd === 'string' && + typeof data.mfnd_opng_dt === 'string' && + typeof data.mfnd_end_dt === 'string' && + typeof data.dpsi_erlm_cncl_dt === 'string' && + typeof data.etf_cu_qty === 'string' && + typeof data.prdt_name === 'string' && + typeof data.prdt_name120 === 'string' && + typeof data.prdt_abrv_name === 'string' && + typeof data.std_pdno === 'string' && + typeof data.prdt_eng_name === 'string' && + typeof data.prdt_eng_name120 === 'string' && + typeof data.prdt_eng_abrv_name === 'string' && + typeof data.dpsi_aptm_erlm_yn === 'string' && + typeof data.etf_txtn_type_cd === 'string' && + typeof data.etf_type_cd === 'string' && + typeof data.lstg_abol_dt === 'string' && + typeof data.nwst_odst_dvsn_cd === 'string' && + typeof data.sbst_pric === 'string' && + typeof data.thco_sbst_pric === 'string' && + typeof data.thco_sbst_pric_chng_dt === 'string' && + typeof data.tr_stop_yn === 'string' && + typeof data.admn_item_yn === 'string' && + typeof data.thdt_clpr === 'string' && + typeof data.bfdy_clpr === 'string' && + typeof data.clpr_chng_dt === 'string' && + typeof data.std_idst_clsf_cd === 'string' && + typeof data.std_idst_clsf_cd_name === 'string' && + typeof data.idx_bztp_lcls_cd_name === 'string' && + typeof data.idx_bztp_mcls_cd_name === 'string' && + typeof data.idx_bztp_scls_cd_name === 'string' && + typeof data.ocr_no === 'string' && + typeof data.crfd_item_yn === 'string' && + typeof data.elec_scty_yn === 'string' && + typeof data.issu_istt_cd === 'string' && + typeof data.etf_chas_erng_rt_dbnb === 'string' && + typeof data.etf_etn_ivst_heed_item_yn === 'string' && + typeof data.stln_int_rt_dvsn_cd === 'string' && + typeof data.frnr_psnl_lmt_rt === 'string' && + typeof data.lstg_rqsr_issu_istt_cd === 'string' && + typeof data.lstg_rqsr_item_cd === 'string' && + typeof data.trst_istt_issu_istt_cd === 'string' + ); +}; + +export type StockDetailQuery = { + pdno: string; + code: string; +}; + +export type FinancialDetail = { + stac_yymm: string; // 결산 년월 + sale_account: string; // 매출액 + sale_cost: string; // 매출원가 + sale_totl_prfi: string; // 매출총이익 + depr_cost: string; // 감가상각비 + sell_mang: string; // 판매관리비 + bsop_prti: string; // 영업이익 + bsop_non_ernn: string; // 영업외수익 + bsop_non_expn: string; // 영업외비용 + op_prfi: string; // 영업이익 + spec_prfi: string; // 특별이익 + spec_loss: string; // 특별손실 + thtr_ntin: string; // 세전순이익 +}; + +export const isFinancialDetail = (data: any): data is FinancialDetail => { + return ( + typeof data.stac_yymm === 'string' && + typeof data.sale_account === 'string' && + typeof data.sale_cost === 'string' && + typeof data.sale_totl_prfi === 'string' && + typeof data.depr_cost === 'string' && + typeof data.sell_mang === 'string' && + typeof data.bsop_prti === 'string' && + typeof data.bsop_non_ernn === 'string' && + typeof data.bsop_non_expn === 'string' && + typeof data.op_prfi === 'string' && + typeof data.spec_prfi === 'string' && + typeof data.spec_loss === 'string' && + typeof data.thtr_ntin === 'string' + ); +}; diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts new file mode 100644 index 00000000..df05e35a --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts @@ -0,0 +1,7 @@ +export type TR_ID = + | 'FHKST03010100' + | 'FHKST66430200' + | 'HHKDB669107C0' + | 'CTPF1002R'; + +export const DEFAULT_TR_ID: TR_ID = 'FHKST03010100'; diff --git a/packages/backend/src/stock/domain/kospiStock.entity.ts b/packages/backend/src/stock/domain/kospiStock.entity.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index 033833bf..24b0fbca 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -1,6 +1,4 @@ import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; -import { DateEmbedded } from '@/common/dateEmbedded.entity'; -import { UserStock } from '@/stock/domain/userStock.entity'; import { StockDaily, StockMinutely, @@ -8,6 +6,8 @@ import { StockWeekly, StockYearly, } from './stockData.entity'; +import { DateEmbedded } from '@/common/dateEmbedded.entity'; +import { UserStock } from '@/stock/domain/userStock.entity'; @Entity() export class Stock { diff --git a/packages/backend/src/stock/domain/stockData.entity.ts b/packages/backend/src/stock/domain/stockData.entity.ts index 55395ffd..f053c2d9 100644 --- a/packages/backend/src/stock/domain/stockData.entity.ts +++ b/packages/backend/src/stock/domain/stockData.entity.ts @@ -1,3 +1,4 @@ +import { applyDecorators } from '@nestjs/common'; import { Entity, PrimaryGeneratedColumn, @@ -8,7 +9,6 @@ import { ColumnOptions, } from 'typeorm'; import { Stock } from './stock.entity'; -import { applyDecorators } from '@nestjs/common'; export const GenerateBigintColumn = ( options?: ColumnOptions, @@ -44,7 +44,7 @@ export class StockData { open: number; @GenerateBigintColumn() - volume: BigInt; + volume: bigint; @Column({ type: 'timestamp', name: 'start_time' }) startTime: Date; From 1e4a4b6db6d2c372fab8b26a0717da26422040e9 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 18 Nov 2024 19:26:09 +0900 Subject: [PATCH 007/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20kospiStock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/stock/domain/kospiStock.entity.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/backend/src/stock/domain/kospiStock.entity.ts b/packages/backend/src/stock/domain/kospiStock.entity.ts index e69de29b..f8578aa7 100644 --- a/packages/backend/src/stock/domain/kospiStock.entity.ts +++ b/packages/backend/src/stock/domain/kospiStock.entity.ts @@ -0,0 +1,13 @@ +import { Column, Entity, OneToOne, PrimaryColumn } from 'typeorm'; +import { Stock } from './stock.entity'; + +@Entity() +export class KospiStock { + @PrimaryColumn({ name: 'stock_id' }) + id: string; + + @Column({ name: 'is_kospi' }) + isKospi: boolean; + + @OneToOne(() => Stock) +} From b003c024d346efe4e5cee609e127d1866c6cad1d Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 18 Nov 2024 21:16:25 +0900 Subject: [PATCH 008/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A3=BC=EC=8B=9D?= =?UTF-8?q?=20=EA=B2=80=EC=83=89=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/stock/dto/stock.Response.ts | 39 +++++++++++++++++++ packages/backend/src/stock/stock.service.ts | 16 +++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/stock/dto/stock.Response.ts b/packages/backend/src/stock/dto/stock.Response.ts index c5139450..6ac76f31 100644 --- a/packages/backend/src/stock/dto/stock.Response.ts +++ b/packages/backend/src/stock/dto/stock.Response.ts @@ -1,5 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; import { Transform } from 'class-transformer'; +import { Stock } from '@/stock/domain/stock.entity'; export class StockViewsResponse { @ApiProperty({ @@ -33,32 +34,70 @@ export class StocksResponse { example: 'A005930', }) id: string; + @ApiProperty({ description: '주식 종목 이름', example: '삼성전자', }) name: string; + @ApiProperty({ description: '주식 현재가', example: 100000.0, }) @Transform(({ value }) => parseFloat(value)) currentPrice: number; + @ApiProperty({ description: '주식 변동률', example: 2.5, }) @Transform(({ value }) => parseFloat(value)) changeRate: number; + @ApiProperty({ description: '주식 거래량', example: 500000, }) @Transform(({ value }) => parseInt(value)) volume: number; + @ApiProperty({ description: '주식 시가 총액', example: '500000000000.00', }) marketCap: string; } + +class StockSearchResult { + @ApiProperty({ + description: '주식 종목 코드', + example: 'A005930', + }) + id: string; + + @ApiProperty({ + description: '주식 종목 이름', + example: '삼성전자', + }) + name: string; +} + +export class StockSearchResponse { + @ApiProperty({ + description: '주식 검색 결과', + type: [StockSearchResult], + }) + searchResults: StockSearchResult[]; + + constructor(stocks?: Stock[]) { + if (!stocks) { + this.searchResults = []; + return; + } + this.searchResults = stocks.map((stock) => ({ + id: stock.id as string, + name: stock.name as string, + })); + } +} diff --git a/packages/backend/src/stock/stock.service.ts b/packages/backend/src/stock/stock.service.ts index 1f23e48f..4b63264a 100644 --- a/packages/backend/src/stock/stock.service.ts +++ b/packages/backend/src/stock/stock.service.ts @@ -3,7 +3,7 @@ import { plainToInstance } from 'class-transformer'; import { DataSource, EntityManager } from 'typeorm'; import { Logger } from 'winston'; import { Stock } from './domain/stock.entity'; -import { StocksResponse } from './dto/stock.Response'; +import { StockSearchResponse, StocksResponse } from './dto/stock.Response'; import { UserStock } from '@/stock/domain/userStock.entity'; @Injectable() @@ -68,6 +68,20 @@ export class StockService { }); } + async searchStock(stockName: string) { + const queryBuilder = this.datasource + .getRepository(Stock) + .createQueryBuilder(); + const result = await queryBuilder + .where('stock.stock_name LIKE :name', { + isTrading: true, + name: `%${stockName}%`, + }) + .limit(10) + .getMany(); + return new StockSearchResponse(result); + } + validateUserStock(userId: number, userStock: UserStock | null) { if (!userStock) { throw new BadRequestException('user stock not found'); From 95af1ae19a279d1845b59594aee56782f6544fb8 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 18 Nov 2024 15:59:19 +0900 Subject: [PATCH 009/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A3=BC=EC=8B=9D?= =?UTF-8?q?=20=EA=B2=80=EC=83=89=20=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8?= =?UTF-8?q?=ED=8A=B8=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/stock/decorator/stock.decorator.ts | 11 +++------- .../backend/src/stock/dto/stock.request.ts | 12 +++++++++++ .../{stock.Response.ts => stock.response.ts} | 0 .../backend/src/stock/stock.controller.ts | 20 ++++++++++++++++++- packages/backend/src/stock/stock.service.ts | 2 +- 5 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/stock/dto/stock.request.ts rename packages/backend/src/stock/dto/{stock.Response.ts => stock.response.ts} (100%) diff --git a/packages/backend/src/stock/decorator/stock.decorator.ts b/packages/backend/src/stock/decorator/stock.decorator.ts index 1412860e..4e2f3469 100644 --- a/packages/backend/src/stock/decorator/stock.decorator.ts +++ b/packages/backend/src/stock/decorator/stock.decorator.ts @@ -1,12 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { - Query, - ParseIntPipe, - DefaultValuePipe, - applyDecorators, -} from '@nestjs/common'; -import { ApiOperation, ApiQuery, ApiResponse } from '@nestjs/swagger'; -import { StocksResponse } from '../dto/stock.Response'; +import { applyDecorators, DefaultValuePipe, ParseIntPipe, Query } from "@nestjs/common"; +import { ApiOperation, ApiQuery, ApiResponse } from "@nestjs/swagger"; +import { StocksResponse } from "../dto/stock.response"; export function LimitQuery(defaultValue = 5): ParameterDecorator { return Query('limit', new DefaultValuePipe(defaultValue), ParseIntPipe); diff --git a/packages/backend/src/stock/dto/stock.request.ts b/packages/backend/src/stock/dto/stock.request.ts new file mode 100644 index 00000000..255e4481 --- /dev/null +++ b/packages/backend/src/stock/dto/stock.request.ts @@ -0,0 +1,12 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class StockSearchRequest { + @ApiProperty({ + description: '검색할 단어', + example: '삼성', + }) + @IsNotEmpty() + @IsString() + name: string; +} diff --git a/packages/backend/src/stock/dto/stock.Response.ts b/packages/backend/src/stock/dto/stock.response.ts similarity index 100% rename from packages/backend/src/stock/dto/stock.Response.ts rename to packages/backend/src/stock/dto/stock.response.ts diff --git a/packages/backend/src/stock/stock.controller.ts b/packages/backend/src/stock/stock.controller.ts index 042353a3..ed85f3c1 100644 --- a/packages/backend/src/stock/stock.controller.ts +++ b/packages/backend/src/stock/stock.controller.ts @@ -32,7 +32,10 @@ import { StockDetailService } from './stockDetail.service'; import SessionGuard from '@/auth/session/session.guard'; import { GetUser } from '@/common/decorator/user.decorator'; import { sessionConfig } from '@/configs/session.config'; -import { StockViewsResponse } from '@/stock/dto/stock.Response'; +import { + StockSearchResponse, + StockViewsResponse, +} from '@/stock/dto/stock.response'; import { StockViewRequest } from '@/stock/dto/stockView.request'; import { UserStockDeleteRequest, @@ -43,6 +46,7 @@ import { UserStockResponse, } from '@/stock/dto/userStock.response'; import { User } from '@/user/domain/user.entity'; +import { StockSearchRequest } from '@/stock/dto/stock.request'; @Controller('stock') export class StockController { @@ -150,6 +154,20 @@ export class StockController { return new UserStockOwnerResponse(result); } + @ApiOperation({ + summary: '주식 검색 API', + description: '주식 이름에 매칭되는 주식을 검색', + }) + @ApiOkResponse({ + description: '검색 완료', + type: StockSearchResponse, + }) + @Get() + async searchStock(@Query() request: StockSearchRequest) { + console.log(request.name); + return await this.stockService.searchStock(request.name); + } + @Get(':stockId/minutely') @ApiGetStockData('주식 분 단위 데이터 조회 API', '분') async getStockDataMinutely( diff --git a/packages/backend/src/stock/stock.service.ts b/packages/backend/src/stock/stock.service.ts index 4b63264a..154eb9d6 100644 --- a/packages/backend/src/stock/stock.service.ts +++ b/packages/backend/src/stock/stock.service.ts @@ -3,7 +3,7 @@ import { plainToInstance } from 'class-transformer'; import { DataSource, EntityManager } from 'typeorm'; import { Logger } from 'winston'; import { Stock } from './domain/stock.entity'; -import { StockSearchResponse, StocksResponse } from './dto/stock.Response'; +import { StockSearchResponse, StocksResponse } from './dto/stock.response'; import { UserStock } from '@/stock/domain/userStock.entity'; @Injectable() From 35a2afc2a6398533d9639cd28bcedb84e5b29674 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 18 Nov 2024 21:25:26 +0900 Subject: [PATCH 010/112] =?UTF-8?q?=E2=9C=A8=20feat:=20swagger=20=EA=B8=B0?= =?UTF-8?q?=EB=B3=B8=20=EA=B2=BD=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/swagger.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/configs/swagger.config.ts b/packages/backend/src/configs/swagger.config.ts index ad58834f..c9d9dfd5 100644 --- a/packages/backend/src/configs/swagger.config.ts +++ b/packages/backend/src/configs/swagger.config.ts @@ -9,5 +9,5 @@ export function useSwagger(app: INestApplication) { .build(); const documentFactory = () => SwaggerModule.createDocument(app, config); - SwaggerModule.setup('swagger', app, documentFactory); + SwaggerModule.setup('api', app, documentFactory); } From 0ab32335143714a10852529c72e55265371818cc Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 18 Nov 2024 22:08:49 +0900 Subject: [PATCH 011/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=97=94=ED=8B=B0=ED=8B=B0=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.module.ts | 3 +- .../backend/src/chat/domain/chat.entity.ts | 5 ++++ .../backend/src/chat/domain/like.entity.ts | 28 +++++++++++++++++++ .../backend/src/stock/domain/stock.entity.ts | 10 +++++-- 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 packages/backend/src/chat/domain/like.entity.ts diff --git a/packages/backend/src/chat/chat.module.ts b/packages/backend/src/chat/chat.module.ts index 089b37e4..4a1a27b5 100644 --- a/packages/backend/src/chat/chat.module.ts +++ b/packages/backend/src/chat/chat.module.ts @@ -5,10 +5,11 @@ import { ChatController } from '@/chat/chat.controller'; import { ChatGateway } from '@/chat/chat.gateway'; import { ChatService } from '@/chat/chat.service'; import { Chat } from '@/chat/domain/chat.entity'; +import { Like } from '@/chat/domain/like.entity'; import { StockModule } from '@/stock/stock.module'; @Module({ - imports: [TypeOrmModule.forFeature([Chat]), StockModule, SessionModule], + imports: [TypeOrmModule.forFeature([Chat, Like]), StockModule, SessionModule], controllers: [ChatController], providers: [ChatGateway, ChatService], }) diff --git a/packages/backend/src/chat/domain/chat.entity.ts b/packages/backend/src/chat/domain/chat.entity.ts index 13800bff..a1bab9ca 100644 --- a/packages/backend/src/chat/domain/chat.entity.ts +++ b/packages/backend/src/chat/domain/chat.entity.ts @@ -3,9 +3,11 @@ import { Entity, JoinColumn, ManyToOne, + OneToMany, PrimaryGeneratedColumn, } from 'typeorm'; import { ChatType } from '@/chat/domain/chatType.enum'; +import { Like } from '@/chat/domain/like.entity'; import { DateEmbedded } from '@/common/dateEmbedded.entity'; import { Stock } from '@/stock/domain/stock.entity'; import { User } from '@/user/domain/user.entity'; @@ -23,6 +25,9 @@ export class Chat { @JoinColumn({ name: 'stock_id' }) stock: Stock; + @OneToMany(() => Like, (like) => like.chat) + likes?: Like[]; + @Column() message: string; diff --git a/packages/backend/src/chat/domain/like.entity.ts b/packages/backend/src/chat/domain/like.entity.ts new file mode 100644 index 00000000..84238b15 --- /dev/null +++ b/packages/backend/src/chat/domain/like.entity.ts @@ -0,0 +1,28 @@ +import { + CreateDateColumn, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { Chat } from '@/chat/domain/chat.entity'; +import { User } from '@/user/domain/user.entity'; + +@Index('chat_user_unique', ['chat', 'user'], { unique: true }) +@Entity() +export class Like { + @PrimaryGeneratedColumn() + id: number; + + @ManyToOne(() => Chat, (chat) => chat.id) + @JoinColumn({ name: 'chat_id' }) + chat: Chat; + + @ManyToOne(() => User, (user) => user.id) + @JoinColumn({ name: 'user_id' }) + user: User; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; +} diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index 033833bf..3645b33e 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -1,6 +1,4 @@ import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; -import { DateEmbedded } from '@/common/dateEmbedded.entity'; -import { UserStock } from '@/stock/domain/userStock.entity'; import { StockDaily, StockMinutely, @@ -8,6 +6,9 @@ import { StockWeekly, StockYearly, } from './stockData.entity'; +import { Like } from '@/chat/domain/like.entity'; +import { DateEmbedded } from '@/common/dateEmbedded.entity'; +import { UserStock } from '@/stock/domain/userStock.entity'; @Entity() export class Stock { @@ -26,8 +27,11 @@ export class Stock { @Column({ name: 'group_code' }) groupCode?: string; + @OneToMany(() => Like, (like) => like.chat) + likes?: Like[]; + @Column(() => DateEmbedded, { prefix: '' }) - dare?: DateEmbedded; + date?: DateEmbedded; @OneToMany(() => UserStock, (userStock) => userStock.stock) userStocks?: UserStock[]; From 74cbf7b5dac3767bafa1f7d02019651b345e2817 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 18 Nov 2024 22:11:06 +0900 Subject: [PATCH 012/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20typeOR?= =?UTF-8?q?M=20=EC=84=A4=EC=A0=95=20=ED=8C=8C=EC=9D=BC=20=EC=9D=B4?= =?UTF-8?q?=EB=A6=84=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/app.module.ts | 2 +- .../src/configs/{devTypeormConfig.ts => typeormConfig.ts} | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) rename packages/backend/src/configs/{devTypeormConfig.ts => typeormConfig.ts} (99%) diff --git a/packages/backend/src/app.module.ts b/packages/backend/src/app.module.ts index d0c5f81d..f4735ad2 100644 --- a/packages/backend/src/app.module.ts +++ b/packages/backend/src/app.module.ts @@ -10,7 +10,7 @@ import { ChatModule } from '@/chat/chat.module'; import { typeormDevelopConfig, typeormProductConfig, -} from '@/configs/devTypeormConfig'; +} from '@/configs/typeormConfig'; import { logger } from '@/configs/logger.config'; import { StockModule } from '@/stock/stock.module'; import { UserModule } from '@/user/user.module'; diff --git a/packages/backend/src/configs/devTypeormConfig.ts b/packages/backend/src/configs/typeormConfig.ts similarity index 99% rename from packages/backend/src/configs/devTypeormConfig.ts rename to packages/backend/src/configs/typeormConfig.ts index 2641d8d9..a8c70a18 100644 --- a/packages/backend/src/configs/devTypeormConfig.ts +++ b/packages/backend/src/configs/typeormConfig.ts @@ -24,4 +24,3 @@ export const typeormDevelopConfig: TypeOrmModuleOptions = { //logging: true, synchronize: true, }; - From 13a8f1f00c277618ef9442b0a28d6be7675f8a73 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Tue, 19 Nov 2024 11:19:59 +0900 Subject: [PATCH 013/112] =?UTF-8?q?=E2=9C=A8=20feat:=20kospi=20stock=20ent?= =?UTF-8?q?ity=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/domain/kospiStock.entity.ts | 3 ++- packages/backend/src/stock/domain/stock.entity.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/stock/domain/kospiStock.entity.ts b/packages/backend/src/stock/domain/kospiStock.entity.ts index f8578aa7..7d96a992 100644 --- a/packages/backend/src/stock/domain/kospiStock.entity.ts +++ b/packages/backend/src/stock/domain/kospiStock.entity.ts @@ -9,5 +9,6 @@ export class KospiStock { @Column({ name: 'is_kospi' }) isKospi: boolean; - @OneToOne(() => Stock) + @OneToOne(() => Stock, (stock) => stock.id) + stock: Stock; } diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index 24b0fbca..463389cf 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -1,4 +1,4 @@ -import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; +import { Column, Entity, OneToMany, OneToOne, PrimaryColumn } from 'typeorm'; import { StockDaily, StockMinutely, @@ -8,6 +8,7 @@ import { } from './stockData.entity'; import { DateEmbedded } from '@/common/dateEmbedded.entity'; import { UserStock } from '@/stock/domain/userStock.entity'; +import { KospiStock } from './kospiStock.entity'; @Entity() export class Stock { @@ -46,4 +47,7 @@ export class Stock { @OneToMany(() => StockYearly, (stockYearly) => stockYearly.stock) stockYearly?: StockYearly[]; + + @OneToOne(() => KospiStock, (kospiStock) => kospiStock.stock) + kospiStock?: KospiStock; } From 17d4cb1b75eeee97126fc299d451d38760af94d6 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Tue, 19 Nov 2024 11:46:35 +0900 Subject: [PATCH 014/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openapi=20detail?= =?UTF-8?q?=20=EC=9E=A0=EA=B9=90=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 18 +++++++++++++++++- .../openapi/type/openapiDetailData.type.ts | 4 ++-- test.js | 11 +++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 643f7636..77fa6feb 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -61,19 +61,26 @@ export class OpenapiDetailData { for (const stock of chunk) { const dataQuery = this.getDetailDataQuery(stock.id!); const defaultQuery = this.getDefaultDataQuery(stock.id!); + // 여기서 가져올 건 eps -> eps와 per 계산하자. const output1 = await getOpenApi(this.incomeUrl, conf, dataQuery, 'FHKST66430200'); + // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 const output2 = await getOpenApi( this.defaultUrl, conf, defaultQuery, 'CTPF1002R', ); + // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 const output3 = await manager.find(StockDaily, { + select: { + + }, where: { } }) - if (isFinancialDetail(output1) && isProductDetail(output2)) { + // 주식 마지막 데이터 끌고 오기. 최신 데이터로. + if ( isProductDetail(output1)) { } } } @@ -99,4 +106,13 @@ export class OpenapiDetailData { fid_div_cls_code: classify, }; } + + private getDate52WeeksAgo(): Date { + const today = new Date(); + const weeksAgo = 52 * 7; + const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); + date52WeeksAgo.setHours(0, 0, 0, 0); + return date52WeeksAgo; + } + } diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts index a4267b40..c5642b3a 100644 --- a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -42,12 +42,12 @@ export type ProductDetail = { scty_grp_id_cd: string; // 증권그룹ID코드 excg_dvsn_cd: string; // 거래소구분코드 setl_mmdd: string; // 결산월일 - lstg_stqt: string; // 상장주수 + lstg_stqt: string; // 상장주수 - 이거 사용 lstg_cptl_amt: string; // 상장자본금액 cpta: string; // 자본금 papr: string; // 액면가 issu_pric: string; // 발행가격 - kospi200_item_yn: string; // 코스피200종목여부 + kospi200_item_yn: string; // 코스피200종목여부 - 이것도 사용 scts_mket_lstg_dt: string; // 유가증권시장상장일자 scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 diff --git a/test.js b/test.js index ec521fac..ea3c8120 100644 --- a/test.js +++ b/test.js @@ -1,5 +1,8 @@ -const getTodayDate = () => { +function getDate52WeeksAgo() { const today = new Date(); - return today.toISOString().split('T')[0].replace(/-/g, ''); -}; -console.log(getTodayDate()); + const weeksAgo = 52 * 7; // 52주 * 7일 + const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); + return date52WeeksAgo; + } + + console.log(getDate52WeeksAgo()); From 1b49e9cf0214d42cc04c7a42f9a17401199210bd Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 12:02:06 +0900 Subject: [PATCH 015/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.module.ts | 3 +- .../backend/src/chat/dto/like.response.ts | 31 +++++++++++++ packages/backend/src/chat/like.service.ts | 46 +++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/chat/dto/like.response.ts create mode 100644 packages/backend/src/chat/like.service.ts diff --git a/packages/backend/src/chat/chat.module.ts b/packages/backend/src/chat/chat.module.ts index 4a1a27b5..62dc1c29 100644 --- a/packages/backend/src/chat/chat.module.ts +++ b/packages/backend/src/chat/chat.module.ts @@ -6,11 +6,12 @@ import { ChatGateway } from '@/chat/chat.gateway'; import { ChatService } from '@/chat/chat.service'; import { Chat } from '@/chat/domain/chat.entity'; import { Like } from '@/chat/domain/like.entity'; +import { LikeService } from '@/chat/like.service'; import { StockModule } from '@/stock/stock.module'; @Module({ imports: [TypeOrmModule.forFeature([Chat, Like]), StockModule, SessionModule], controllers: [ChatController], - providers: [ChatGateway, ChatService], + providers: [ChatGateway, ChatService, LikeService], }) export class ChatModule {} diff --git a/packages/backend/src/chat/dto/like.response.ts b/packages/backend/src/chat/dto/like.response.ts new file mode 100644 index 00000000..94fafde6 --- /dev/null +++ b/packages/backend/src/chat/dto/like.response.ts @@ -0,0 +1,31 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class LikeResponse { + @ApiProperty({ + type: Number, + description: '좋아요를 누른 채팅의 ID', + example: 1, + }) + chatId: number; + + @ApiProperty({ + type: Number, + description: '채팅의 좋아요 수', + example: 45, + }) + likeCount: number; + + @ApiProperty({ + type: String, + description: '결과 메시지', + example: 'like chat', + }) + message: string; + + @ApiProperty({ + type: Date, + description: '좋아요를 누른 시간', + example: '2021-08-01T00:00:00', + }) + date: Date; +} diff --git a/packages/backend/src/chat/like.service.ts b/packages/backend/src/chat/like.service.ts new file mode 100644 index 00000000..a58a0104 --- /dev/null +++ b/packages/backend/src/chat/like.service.ts @@ -0,0 +1,46 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { DataSource, EntityManager } from 'typeorm'; +import { Chat } from '@/chat/domain/chat.entity'; +import { Like } from '@/chat/domain/like.entity'; +import { LikeResponse } from '@/chat/dto/like.response'; + +@Injectable() +export class LikeService { + constructor(private readonly dataSource: DataSource) {} + + async toggleLike(userId: number, chatId: number) { + return await this.dataSource.transaction(async (manager) => { + const chat = await this.findChat(chatId, manager); + return await this.saveLike(manager, chat, userId); + }); + } + + private async findChat(chatId: number, manager: EntityManager) { + const chat = await manager.findOne(Chat, { where: { id: chatId } }); + if (!chat) { + throw new BadRequestException('Chat not found'); + } + return chat; + } + + private async saveLike( + manager: EntityManager, + chat: Chat, + userId: number, + ): Promise { + chat.likeCount += 1; + await Promise.all([ + manager.save(Like, { + user: { id: userId }, + chat, + }), + manager.save(Chat, chat), + ]); + return { + likeCount: chat.likeCount, + message: 'like chat', + chatId: chat.id, + date: chat.date.updatedAt, + }; + } +} From 011f4f0379ab6869e4f76218dbb0e31cf3e132d8 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 12:03:18 +0900 Subject: [PATCH 016/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=97=94=EB=93=9C=20=ED=8F=AC=EC=9D=B8=ED=8A=B8=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 20 ++++++++++-- .../src/chat/decorator/like.decorator.ts | 31 +++++++++++++++++++ packages/backend/src/chat/dto/like.request.ts | 13 ++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/chat/decorator/like.decorator.ts create mode 100644 packages/backend/src/chat/dto/like.request.ts diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index f2a5452d..4e2c9371 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -1,16 +1,25 @@ -import { Controller, Get, Query } from '@nestjs/common'; +import { Body, Controller, Get, Post, Query, UseGuards } from '@nestjs/common'; import { ApiBadRequestResponse, ApiOkResponse, ApiOperation, } from '@nestjs/swagger'; +import SessionGuard from '@/auth/session/session.guard'; import { ChatService } from '@/chat/chat.service'; +import { ToggleLikeApi } from '@/chat/decorator/like.decorator'; import { ChatScrollRequest } from '@/chat/dto/chat.request'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; +import { LikeRequest } from '@/chat/dto/like.request'; +import { LikeService } from '@/chat/like.service'; +import { GetUser } from '@/common/decorator/user.decorator'; +import { User } from '@/user/domain/user.entity'; @Controller('chat') export class ChatController { - constructor(private readonly chatService: ChatService) {} + constructor( + private readonly chatService: ChatService, + private readonly likeService: LikeService, + ) {} @ApiOperation({ summary: '채팅 스크롤 조회 API', @@ -36,4 +45,11 @@ export class ChatController { request.pageSize, ); } + + @UseGuards(SessionGuard) + @ToggleLikeApi() + @Post('like') + async toggleChatLike(@Body() request: LikeRequest, @GetUser() user: User) { + return await this.likeService.toggleLike(user.id, request.chatId); + } } diff --git a/packages/backend/src/chat/decorator/like.decorator.ts b/packages/backend/src/chat/decorator/like.decorator.ts new file mode 100644 index 00000000..e1a1ea6b --- /dev/null +++ b/packages/backend/src/chat/decorator/like.decorator.ts @@ -0,0 +1,31 @@ +import { applyDecorators } from '@nestjs/common'; +import { + ApiBadRequestResponse, + ApiCookieAuth, + ApiOkResponse, + ApiOperation, +} from '@nestjs/swagger'; +import { LikeResponse } from '@/chat/dto/like.response'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export function ToggleLikeApi() { + return applyDecorators( + ApiCookieAuth(), + ApiOperation({ + summary: '채팅 좋아요 토글 API', + description: '채팅 좋아요를 토글한다.', + }), + ApiOkResponse({ + description: '좋아요 성공', + type: LikeResponse, + }), + ApiBadRequestResponse({ + description: '채팅이 존재하지 않음', + example: { + message: 'Chat not found', + error: 'Bad Request', + statusCode: 400, + }, + }), + ); +} diff --git a/packages/backend/src/chat/dto/like.request.ts b/packages/backend/src/chat/dto/like.request.ts new file mode 100644 index 00000000..02feec50 --- /dev/null +++ b/packages/backend/src/chat/dto/like.request.ts @@ -0,0 +1,13 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNumber } from 'class-validator'; + +export class LikeRequest { + @ApiProperty({ + required: true, + type: Number, + description: '좋아요를 누를 채팅의 ID', + example: 1, + }) + @IsNumber() + chatId: number; +} From 71d51155be7a6a7480ae5e87b7f5e2de7057cc0b Mon Sep 17 00:00:00 2001 From: sunghwki Date: Tue, 19 Nov 2024 13:25:46 +0900 Subject: [PATCH 017/112] =?UTF-8?q?=F0=9F=92=84=20style:=20=EC=95=88=20?= =?UTF-8?q?=EC=93=B0=EC=9D=B4=EB=8A=94=20bigint=20=EB=8D=B0=EC=BD=94?= =?UTF-8?q?=EB=A0=88=EC=9D=B4=ED=84=B0=20=EB=B0=8F=20=EA=B4=80=EB=A0=A8=20?= =?UTF-8?q?import=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/stock/domain/stockData.entity.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/packages/backend/src/stock/domain/stockData.entity.ts b/packages/backend/src/stock/domain/stockData.entity.ts index e9c75407..7f6790d3 100644 --- a/packages/backend/src/stock/domain/stockData.entity.ts +++ b/packages/backend/src/stock/domain/stockData.entity.ts @@ -1,4 +1,3 @@ -import { applyDecorators } from '@nestjs/common'; import { Entity, PrimaryGeneratedColumn, @@ -6,27 +5,9 @@ import { CreateDateColumn, JoinColumn, ManyToOne, - ColumnOptions, } from 'typeorm'; import { Stock } from './stock.entity'; -export const GenerateBigintColumn = ( - options?: ColumnOptions, -): PropertyDecorator => { - return applyDecorators( - Column({ - ...options, - type: 'bigint', - transformer: { - to: (value: bigint): string => - typeof value === 'bigint' ? value.toString() : value, - from: (value: string): bigint => - typeof value === 'string' ? BigInt(value) : value, - }, - }), - ); -}; - export class StockData { @PrimaryGeneratedColumn() id: number; From 8dc2acf226678c0e6dd16c310eae8a59e552cc6b Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 14:10:29 +0900 Subject: [PATCH 018/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=B7=A8=EC=86=8C=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/chat/like.service.spec.ts | 68 +++++++++++++++++++ packages/backend/src/chat/like.service.ts | 21 ++++++ 2 files changed, 89 insertions(+) create mode 100644 packages/backend/src/chat/like.service.spec.ts diff --git a/packages/backend/src/chat/like.service.spec.ts b/packages/backend/src/chat/like.service.spec.ts new file mode 100644 index 00000000..1099b7bd --- /dev/null +++ b/packages/backend/src/chat/like.service.spec.ts @@ -0,0 +1,68 @@ +import { createDataSourceMock } from '@/user/user.service.spec'; +import { DataSource } from 'typeorm'; +import { LikeService } from '@/chat/like.service'; +import { Chat } from '@/chat/domain/chat.entity'; +import { Stock } from '@/stock/domain/stock.entity'; +import { User } from '@/user/domain/user.entity'; +import { Like } from '@/chat/domain/like.entity'; + +function createChat(): Chat { + return { + stock: new Stock(), + user: new User(), + id: 1, + likeCount: 1, + message: '안녕하세요', + type: 'NORMAL', + date: { + createdAt: new Date(), + updatedAt: new Date(), + }, + }; +} + +describe('LikeService 테스트', () => { + test('존재하지 않는 채팅을 좋아요를 시도하면 예외가 발생한다.', () => { + const managerMock = { + findOne: jest.fn().mockResolvedValue(null), + }; + const datasource = createDataSourceMock(managerMock); + const likeService = new LikeService(datasource as DataSource); + + expect(likeService.toggleLike(1, 1)).rejects.toThrow('Chat not found'); + }); + + test('특정 채팅에 좋아요를 한다.', async () => { + const chat = createChat(); + const managerMock = { + findOne: jest + .fn() + .mockResolvedValueOnce(chat) + .mockResolvedValueOnce(null), + save: jest.fn(), + }; + const datasource = createDataSourceMock(managerMock); + const likeService = new LikeService(datasource as DataSource); + + const response = await likeService.toggleLike(1, 1); + + expect(response.likeCount).toBe(2); + }); + + test('특정 채팅에 좋아요를 취소한다.', async () => { + const chat = createChat(); + const managerMock = { + findOne: jest + .fn() + .mockResolvedValueOnce(chat) + .mockResolvedValueOnce(new Like()), + remove: jest.fn(), + }; + const datasource = createDataSourceMock(managerMock); + const likeService = new LikeService(datasource as DataSource); + + const response = await likeService.toggleLike(1, 1); + + expect(response.likeCount).toBe(0); + }); +}); \ No newline at end of file diff --git a/packages/backend/src/chat/like.service.ts b/packages/backend/src/chat/like.service.ts index a58a0104..014111a7 100644 --- a/packages/backend/src/chat/like.service.ts +++ b/packages/backend/src/chat/like.service.ts @@ -11,6 +11,12 @@ export class LikeService { async toggleLike(userId: number, chatId: number) { return await this.dataSource.transaction(async (manager) => { const chat = await this.findChat(chatId, manager); + const like = await manager.findOne(Like, { + where: { user: { id: userId }, chat: { id: chatId } }, + }); + if (like) { + return await this.deleteLike(manager, chat, like); + } return await this.saveLike(manager, chat, userId); }); } @@ -43,4 +49,19 @@ export class LikeService { date: chat.date.updatedAt, }; } + + private async deleteLike( + manager: EntityManager, + chat: Chat, + like: Like, + ): Promise { + chat.likeCount -= 1; + await Promise.all([manager.remove(like), manager.save(Chat, chat)]); + return { + likeCount: chat.likeCount, + message: 'like cancel', + chatId: chat.id, + date: chat.date.updatedAt, + }; + } } From 3a5e55239b48d56f0bebd4d1525e2ea4edda2f2b Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 16:56:00 +0900 Subject: [PATCH 019/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=9D=B4=EB=B2=A4=ED=8A=B8=EB=A5=BC=20=EC=B0=B8?= =?UTF-8?q?=EC=97=AC=EC=A4=91=EC=9D=B8=20=EC=B1=84=ED=8C=85=EB=B0=A9?= =?UTF-8?q?=EC=97=90=20=EC=A0=84=ED=8C=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 6 ++- packages/backend/src/chat/chat.gateway.ts | 5 +++ .../backend/src/chat/dto/like.response.ts | 38 +++++++++++++++++++ packages/backend/src/chat/like.service.ts | 19 +++------- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index 4e2c9371..ddcfc87f 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -13,12 +13,14 @@ import { LikeRequest } from '@/chat/dto/like.request'; import { LikeService } from '@/chat/like.service'; import { GetUser } from '@/common/decorator/user.decorator'; import { User } from '@/user/domain/user.entity'; +import { ChatGateway } from '@/chat/chat.gateway'; @Controller('chat') export class ChatController { constructor( private readonly chatService: ChatService, private readonly likeService: LikeService, + private readonly chatGateWay: ChatGateway, ) {} @ApiOperation({ @@ -50,6 +52,8 @@ export class ChatController { @ToggleLikeApi() @Post('like') async toggleChatLike(@Body() request: LikeRequest, @GetUser() user: User) { - return await this.likeService.toggleLike(user.id, request.chatId); + const result = await this.likeService.toggleLike(user.id, request.chatId); + this.chatGateWay.broadcastLike(result); + return result; } } diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index f5fc099e..768fe383 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -17,6 +17,7 @@ import { ChatService } from '@/chat/chat.service'; import { Chat } from '@/chat/domain/chat.entity'; import { WebSocketExceptionFilter } from '@/middlewares/filter/webSocketException.filter'; import { StockService } from '@/stock/stock.service'; +import { LikeResponse } from '@/chat/dto/like.response'; interface chatMessage { room: string; @@ -65,6 +66,10 @@ export class ChatGateway implements OnGatewayConnection { this.server.to(room).emit('chat', this.toResponse(savedChat)); } + async broadcastLike(response: LikeResponse) { + this.server.to(response.stockId).emit('like', response); + } + async handleConnection(client: Socket) { const room = client.handshake.query.stockId; if ( diff --git a/packages/backend/src/chat/dto/like.response.ts b/packages/backend/src/chat/dto/like.response.ts index 94fafde6..5c6ff7fa 100644 --- a/packages/backend/src/chat/dto/like.response.ts +++ b/packages/backend/src/chat/dto/like.response.ts @@ -1,4 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; +import { Chat } from '@/chat/domain/chat.entity'; export class LikeResponse { @ApiProperty({ @@ -8,6 +9,13 @@ export class LikeResponse { }) chatId: number; + @ApiProperty({ + type: 'string', + description: '참여 중인 좀목 id', + example: 'A005930', + }) + stockId: string; + @ApiProperty({ type: Number, description: '채팅의 좋아요 수', @@ -28,4 +36,34 @@ export class LikeResponse { example: '2021-08-01T00:00:00', }) date: Date; + + static createLikeResponse(chat: Chat): LikeResponse { + if (!isStockId(chat.stock.id)) { + throw new Error(`Stock id is undefined: ${chat.id}`); + } + return { + stockId: chat.stock.id, + chatId: chat.id, + likeCount: chat.likeCount, + message: 'like chat', + date: chat.date.updatedAt, + }; + } + + static createUnlikeResponse(chat: Chat): LikeResponse { + if (!isStockId(chat.stock.id)) { + throw new Error(`Stock id is undefined: ${chat.id}`); + } + return { + stockId: chat.stock.id, + chatId: chat.id, + likeCount: chat.likeCount, + message: 'like cancel', + date: chat.date.updatedAt, + }; + } } + +function isStockId(stockId?: string): stockId is string { + return stockId !== undefined; +} \ No newline at end of file diff --git a/packages/backend/src/chat/like.service.ts b/packages/backend/src/chat/like.service.ts index 014111a7..a1495905 100644 --- a/packages/backend/src/chat/like.service.ts +++ b/packages/backend/src/chat/like.service.ts @@ -22,7 +22,10 @@ export class LikeService { } private async findChat(chatId: number, manager: EntityManager) { - const chat = await manager.findOne(Chat, { where: { id: chatId } }); + const chat = await manager.findOne(Chat, { + where: { id: chatId }, + relations: ['stock'], + }); if (!chat) { throw new BadRequestException('Chat not found'); } @@ -42,12 +45,7 @@ export class LikeService { }), manager.save(Chat, chat), ]); - return { - likeCount: chat.likeCount, - message: 'like chat', - chatId: chat.id, - date: chat.date.updatedAt, - }; + return LikeResponse.createLikeResponse(chat); } private async deleteLike( @@ -57,11 +55,6 @@ export class LikeService { ): Promise { chat.likeCount -= 1; await Promise.all([manager.remove(like), manager.save(Chat, chat)]); - return { - likeCount: chat.likeCount, - message: 'like cancel', - chatId: chat.id, - date: chat.date.updatedAt, - }; + return LikeResponse.createUnlikeResponse(chat); } } From e26fe57a7391154e1fc0ba2a6b589f59514c983a Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 20:14:13 +0900 Subject: [PATCH 020/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=B1=84=ED=8C=85?= =?UTF-8?q?=20=EC=8A=A4=ED=81=AC=EB=A1=A4=20=EC=8B=9C=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=97=AC=EB=B6=80=20=EC=B6=9C=EB=A0=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 22 ++++--- packages/backend/src/chat/chat.gateway.ts | 46 ++++++++++----- .../backend/src/chat/chat.service.spec.ts | 8 ++- packages/backend/src/chat/chat.service.ts | 58 +++++++++++-------- packages/backend/src/chat/dto/chat.request.ts | 25 ++++++++ .../backend/src/chat/dto/chat.response.ts | 3 + 6 files changed, 114 insertions(+), 48 deletions(-) diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index ddcfc87f..84c30070 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -1,4 +1,12 @@ -import { Body, Controller, Get, Post, Query, UseGuards } from '@nestjs/common'; +import { + Body, + Controller, + Get, + Post, + Query, + Req, + UseGuards, +} from '@nestjs/common'; import { ApiBadRequestResponse, ApiOkResponse, @@ -40,12 +48,12 @@ export class ChatController { }, }) @Get() - async findChatList(@Query() request: ChatScrollRequest) { - return await this.chatService.scrollNextChat( - request.stockId, - request.latestChatId, - request.pageSize, - ); + async findChatList( + @Query() request: ChatScrollRequest, + @Req() req: Express.Request, + ) { + const user = req.user as User; + return await this.chatService.scrollNextChat(request, user?.id); } @UseGuards(SessionGuard) diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index 768fe383..ad600a47 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -18,6 +18,7 @@ import { Chat } from '@/chat/domain/chat.entity'; import { WebSocketExceptionFilter } from '@/middlewares/filter/webSocketException.filter'; import { StockService } from '@/stock/stock.service'; import { LikeResponse } from '@/chat/dto/like.response'; +import { ChatScrollQuery, isChatScrollQuery } from '@/chat/dto/chat.request'; interface chatMessage { room: string; @@ -36,6 +37,7 @@ interface chatResponse { export class ChatGateway implements OnGatewayConnection { @WebSocketServer() server: Server; + constructor( @Inject('winston') private readonly logger: Logger, private readonly stockService: StockService, @@ -71,24 +73,40 @@ export class ChatGateway implements OnGatewayConnection { } async handleConnection(client: Socket) { - const room = client.handshake.query.stockId; - if ( - !this.isString(room) || - !(await this.stockService.checkStockExist(room)) - ) { - client.emit('error', 'Invalid stockId'); - this.logger.warn(`client connected with invalid stockId: ${room}`); + try { + const { stockId, pageSize } = await this.getChatScrollQuery(client); + await this.validateExistStock(stockId); + client.join(stockId); + const messages = await this.chatService.scrollFirstChat({ + stockId, + pageSize, + }); + this.logger.info(`client joined room ${stockId}`); + client.emit('chat', messages); + } catch (e) { + const error = e as Error; + this.logger.warn(error.message); + client.emit('error', error.message); client.disconnect(); - return; } - client.join(room); - const messages = await this.chatService.scrollFirstChat(room); - this.logger.info(`client joined room ${room}`); - client.emit('chat', messages); } - private isString(value: string | string[] | undefined): value is string { - return typeof value === 'string'; + private async validateExistStock(stockId: string): Promise { + if (!(await this.stockService.checkStockExist(stockId))) { + throw new Error(`Stock does not exist: ${stockId}`); + } + } + + private async getChatScrollQuery(client: Socket): Promise { + const query = client.handshake.query; + if (!isChatScrollQuery(query)) { + throw new Error('Invalid chat scroll query'); + } + return { + stockId: query.stockId, + latestChatId: query.latestChatId, + pageSize: query.pageSize, + }; } private toResponse(chat: Chat): chatResponse { diff --git a/packages/backend/src/chat/chat.service.spec.ts b/packages/backend/src/chat/chat.service.spec.ts index 7d181061..24c77fae 100644 --- a/packages/backend/src/chat/chat.service.spec.ts +++ b/packages/backend/src/chat/chat.service.spec.ts @@ -8,7 +8,11 @@ describe('ChatService 테스트', () => { const chatService = new ChatService(dataSource as DataSource); await expect(() => - chatService.scrollNextChat('A005930', 1, 101), + chatService.scrollNextChat({ + stockId: 'A005930', + latestChatId: 1, + pageSize: 101, + }), ).rejects.toThrow('pageSize should be less than 100'); }); @@ -17,7 +21,7 @@ describe('ChatService 테스트', () => { const chatService = new ChatService(dataSource as DataSource); await expect(() => - chatService.scrollFirstChat('A005930', 101), + chatService.scrollFirstChat({ stockId: 'A005930', pageSize: 101 }), ).rejects.toThrow('pageSize should be less than 100'); }); }); diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index ef262e7b..18230c2c 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -2,6 +2,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; +import { ChatScrollQuery } from '@/chat/dto/chat.request'; export interface ChatMessage { message: string; @@ -22,19 +23,17 @@ export class ChatService { }); } - async scrollFirstChat(stockId: string, scrollSize?: number) { - this.validatePageSize(scrollSize); - const result = await this.findFirstChatScroll(stockId, scrollSize); - return await this.toScrollResponse(result, scrollSize); + async scrollFirstChat(chatScrollQuery: ChatScrollQuery, userId?: number) { + const { pageSize } = chatScrollQuery; + this.validatePageSize(pageSize); + const result = await this.findFirstChatScroll(chatScrollQuery, userId); + return await this.toScrollResponse(result, pageSize); } - async scrollNextChat( - stockId: string, - latestChatId?: number, - pageSize?: number, - ) { + async scrollNextChat(chatScrollQuery: ChatScrollQuery, userId?: number) { + const { pageSize } = chatScrollQuery; this.validatePageSize(pageSize); - const result = await this.findChatScroll(stockId, latestChatId, pageSize); + const result = await this.findChatScroll(chatScrollQuery, userId); return await this.toScrollResponse(result, pageSize); } @@ -54,39 +53,48 @@ export class ChatService { } private async findChatScroll( - stockId: string, - latestChatId?: number, - pageSize?: number, + chatScrollQuery: ChatScrollQuery, + userId?: number, ) { - if (!latestChatId) { - return await this.findFirstChatScroll(stockId, pageSize); + if (!chatScrollQuery.latestChatId) { + return await this.findFirstChatScroll(chatScrollQuery, userId); } else { - return await this.findNextChatScroll(stockId, latestChatId, pageSize); + return await this.findNextChatScroll(chatScrollQuery); } } - private async findFirstChatScroll(stockId: string, pageSize?: number) { + private async findFirstChatScroll( + chatScrollQuery: ChatScrollQuery, + userId?: number, + ) { const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - if (!pageSize) { - pageSize = DEFAULT_PAGE_SIZE; + if (!chatScrollQuery.pageSize) { + chatScrollQuery.pageSize = DEFAULT_PAGE_SIZE; } + const { stockId, pageSize } = chatScrollQuery; return queryBuilder + .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { + userId, + }) .where('chat.stock_id = :stockId', { stockId }) .orderBy('chat.id', 'DESC') - .limit(pageSize + 1) + .take(pageSize + 1) .getMany(); } private async findNextChatScroll( - stockId: string, - latestChatId: number, - pageSize?: number, + chatScrollQuery: ChatScrollQuery, + userId?: number, ) { const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - if (!pageSize) { - pageSize = DEFAULT_PAGE_SIZE; + if (!chatScrollQuery.pageSize) { + chatScrollQuery.pageSize = DEFAULT_PAGE_SIZE; } + const { stockId, latestChatId, pageSize } = chatScrollQuery; return queryBuilder + .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { + userId, + }) .where('chat.stock_id = :stockId and chat.id < :latestChatId', { stockId, latestChatId, diff --git a/packages/backend/src/chat/dto/chat.request.ts b/packages/backend/src/chat/dto/chat.request.ts index f3260509..35efb3ad 100644 --- a/packages/backend/src/chat/dto/chat.request.ts +++ b/packages/backend/src/chat/dto/chat.request.ts @@ -28,3 +28,28 @@ export class ChatScrollRequest { @IsNumber() readonly pageSize?: number; } + +export interface ChatScrollQuery { + stockId: string; + latestChatId?: number; + pageSize?: number; +} + +export function isChatScrollQuery(object: unknown): object is ChatScrollQuery { + if (typeof object !== 'object' || object === null) { + return false; + } + + if (!('stockId' in object) || typeof object.stockId !== 'string') { + return false; + } + + if ( + 'latestChatId' in object && + !Number.isInteger(Number(object.latestChatId)) + ) { + return false; + } + + return !('pageSize' in object && !Number.isInteger(Number(object.pageSize))); +} diff --git a/packages/backend/src/chat/dto/chat.response.ts b/packages/backend/src/chat/dto/chat.response.ts index 48aacb0d..803b81e6 100644 --- a/packages/backend/src/chat/dto/chat.response.ts +++ b/packages/backend/src/chat/dto/chat.response.ts @@ -7,6 +7,7 @@ interface ChatResponse { likeCount: number; message: string; type: string; + liked: boolean; createdAt: Date; } @@ -25,6 +26,7 @@ export class ChatScrollResponse { likeCount: 0, message: '안녕하세요', type: ChatType.NORMAL, + isLiked: true, createdAt: new Date(), }, ], @@ -38,6 +40,7 @@ export class ChatScrollResponse { message: chat.message, type: chat.type, createdAt: chat.date!.createdAt, + liked: !!(chat.likes && chat.likes.length > 0), })); this.hasMore = hasMore; } From 28a201b39331f6784a5814efa2f4795803a768e4 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 22:19:21 +0900 Subject: [PATCH 021/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20stock?= =?UTF-8?q?=20=EB=82=B4=EB=B6=80=20=ED=95=84=EB=93=9C=20=ED=83=80=EC=9E=85?= =?UTF-8?q?=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/domain/stock.entity.ts | 12 ++++++------ packages/backend/src/stock/stock.controller.ts | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index 3645b33e..429e8812 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -13,10 +13,10 @@ import { UserStock } from '@/stock/domain/userStock.entity'; @Entity() export class Stock { @PrimaryColumn({ name: 'stock_id' }) - id?: string; + id: string; @Column({ name: 'stock_name' }) - name?: string; + name: string; @Column({ default: 0 }) views: number = 0; @@ -25,14 +25,14 @@ export class Stock { isTrading: boolean = true; @Column({ name: 'group_code' }) - groupCode?: string; - - @OneToMany(() => Like, (like) => like.chat) - likes?: Like[]; + groupCode: string; @Column(() => DateEmbedded, { prefix: '' }) date?: DateEmbedded; + @OneToMany(() => Like, (like) => like.chat) + likes?: Like[]; + @OneToMany(() => UserStock, (userStock) => userStock.stock) userStocks?: UserStock[]; diff --git a/packages/backend/src/stock/stock.controller.ts b/packages/backend/src/stock/stock.controller.ts index ed85f3c1..5aaf6707 100644 --- a/packages/backend/src/stock/stock.controller.ts +++ b/packages/backend/src/stock/stock.controller.ts @@ -164,7 +164,6 @@ export class StockController { }) @Get() async searchStock(@Query() request: StockSearchRequest) { - console.log(request.name); return await this.stockService.searchStock(request.name); } From afbf0b3c1f3db4b1923d63f400339f9da32e0724 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 22:23:37 +0900 Subject: [PATCH 022/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20chat?= =?UTF-8?q?=20=EC=8A=A4=ED=81=AC=EB=A1=A4=20=EC=BF=BC=EB=A6=AC=20=EC=A4=91?= =?UTF-8?q?=EB=B3=B5=20dto=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 6 +++--- packages/backend/src/chat/chat.service.ts | 2 +- packages/backend/src/chat/dto/chat.request.ts | 12 +++--------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index 84c30070..d295f0c9 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -13,15 +13,15 @@ import { ApiOperation, } from '@nestjs/swagger'; import SessionGuard from '@/auth/session/session.guard'; +import { ChatGateway } from '@/chat/chat.gateway'; import { ChatService } from '@/chat/chat.service'; import { ToggleLikeApi } from '@/chat/decorator/like.decorator'; -import { ChatScrollRequest } from '@/chat/dto/chat.request'; +import { ChatScrollQuery } from '@/chat/dto/chat.request'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; import { LikeRequest } from '@/chat/dto/like.request'; import { LikeService } from '@/chat/like.service'; import { GetUser } from '@/common/decorator/user.decorator'; import { User } from '@/user/domain/user.entity'; -import { ChatGateway } from '@/chat/chat.gateway'; @Controller('chat') export class ChatController { @@ -49,7 +49,7 @@ export class ChatController { }) @Get() async findChatList( - @Query() request: ChatScrollRequest, + @Query() request: ChatScrollQuery, @Req() req: Express.Request, ) { const user = req.user as User; diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 18230c2c..650c5fb4 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -1,8 +1,8 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; -import { ChatScrollResponse } from '@/chat/dto/chat.response'; import { ChatScrollQuery } from '@/chat/dto/chat.request'; +import { ChatScrollResponse } from '@/chat/dto/chat.response'; export interface ChatMessage { message: string; diff --git a/packages/backend/src/chat/dto/chat.request.ts b/packages/backend/src/chat/dto/chat.request.ts index 35efb3ad..0c68943b 100644 --- a/packages/backend/src/chat/dto/chat.request.ts +++ b/packages/backend/src/chat/dto/chat.request.ts @@ -1,13 +1,13 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsNumber, IsOptional, IsString } from 'class-validator'; -export class ChatScrollRequest { +export class ChatScrollQuery { @ApiProperty({ description: '종목 주식 id(종목방 id)', example: 'A005930', }) @IsString() - readonly stockId: string; + stockId: string; @ApiProperty({ description: '최신 채팅 id', @@ -16,7 +16,7 @@ export class ChatScrollRequest { }) @IsOptional() @IsNumber() - readonly latestChatId?: number; + latestChatId?: number; @ApiProperty({ description: '페이지 크기', @@ -26,12 +26,6 @@ export class ChatScrollRequest { }) @IsOptional() @IsNumber() - readonly pageSize?: number; -} - -export interface ChatScrollQuery { - stockId: string; - latestChatId?: number; pageSize?: number; } From 5efb9137b905e7f35771ef7005f9c94b5cc009b3 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 23:26:51 +0900 Subject: [PATCH 023/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=9B=B9=EC=86=8C?= =?UTF-8?q?=EC=BC=93=20=EC=B4=88=EA=B8=B0=20=EC=A0=91=EC=86=8D=20=EC=84=B8?= =?UTF-8?q?=EC=85=98=20=EC=9D=B8=EC=A6=9D=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../auth/google/googleAuth.service.spec.ts | 2 +- packages/backend/src/auth/session.module.ts | 2 +- .../auth/session/webSocketSession.guard..ts | 2 +- .../auth/session/websocketSession.service.ts | 25 +++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 packages/backend/src/auth/session/websocketSession.service.ts diff --git a/packages/backend/src/auth/google/googleAuth.service.spec.ts b/packages/backend/src/auth/google/googleAuth.service.spec.ts index 3d3f49be..1e714846 100644 --- a/packages/backend/src/auth/google/googleAuth.service.spec.ts +++ b/packages/backend/src/auth/google/googleAuth.service.spec.ts @@ -15,7 +15,7 @@ describe('GoogleAuthService 테스트', () => { }; test('oauthId와 type에 맞는 유저가 있으면 해당 객체를 반환한다.', async () => { - const user: User = { + const user: Partial = { id: 1, role: Role.USER, type: OauthType.GOOGLE, diff --git a/packages/backend/src/auth/session.module.ts b/packages/backend/src/auth/session.module.ts index 07c36bba..ddd58c70 100644 --- a/packages/backend/src/auth/session.module.ts +++ b/packages/backend/src/auth/session.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; import { MemoryStore } from 'express-session'; -export const MEMORY_STORE = 'memoryStore'; +export const MEMORY_STORE = Symbol('memoryStore'); @Module({ providers: [ diff --git a/packages/backend/src/auth/session/webSocketSession.guard..ts b/packages/backend/src/auth/session/webSocketSession.guard..ts index a19ff956..f2c766a3 100644 --- a/packages/backend/src/auth/session/webSocketSession.guard..ts +++ b/packages/backend/src/auth/session/webSocketSession.guard..ts @@ -15,7 +15,7 @@ export interface SessionSocket extends Socket { session?: User; } -interface PassportSession extends SessionData { +export interface PassportSession extends SessionData { passport: { user: User }; } diff --git a/packages/backend/src/auth/session/websocketSession.service.ts b/packages/backend/src/auth/session/websocketSession.service.ts new file mode 100644 index 00000000..edfd8ea9 --- /dev/null +++ b/packages/backend/src/auth/session/websocketSession.service.ts @@ -0,0 +1,25 @@ +import { MemoryStore } from 'express-session'; +import { Socket } from 'socket.io'; +import { websocketCookieParse } from '@/auth/session/cookieParser'; +import { PassportSession } from '@/auth/session/webSocketSession.guard.'; + +export class WebsocketSessionService { + constructor(private readonly sessionStore: MemoryStore) {} + + async getAuthenticatedUser(socket: Socket) { + const cookieValue = websocketCookieParse(socket); + const session = await this.getSession(cookieValue); + return session ? session.passport.user : undefined; + } + + private getSession(cookieValue: string) { + return new Promise((resolve) => { + this.sessionStore.get(cookieValue, (err: Error, session) => { + if (err || !session) { + resolve(undefined); + } + resolve(session as PassportSession); + }); + }); + } +} From 6c90439809e6ba6c80c1bc585c7911e73f23c47b Mon Sep 17 00:00:00 2001 From: kimminsu Date: Tue, 19 Nov 2024 23:28:04 +0900 Subject: [PATCH 024/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=B1=84=ED=8C=85?= =?UTF-8?q?=EB=B0=A9=20=EC=A0=91=EC=86=8D=20=EC=8B=9C=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=97=AC=EB=B6=80=EB=8F=84=20=EC=B6=9C=EB=A0=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.gateway.ts | 26 +++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index ad600a47..21ba2290 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -7,18 +7,21 @@ import { WebSocketGateway, WebSocketServer, } from '@nestjs/websockets'; +import { MemoryStore } from 'express-session'; import { Server, Socket } from 'socket.io'; import { Logger } from 'winston'; import { SessionSocket, WebSocketSessionGuard, } from '@/auth/session/webSocketSession.guard.'; +import { WebsocketSessionService } from '@/auth/session/websocketSession.service'; +import { MEMORY_STORE } from '@/auth/session.module'; import { ChatService } from '@/chat/chat.service'; import { Chat } from '@/chat/domain/chat.entity'; +import { ChatScrollQuery, isChatScrollQuery } from '@/chat/dto/chat.request'; +import { LikeResponse } from '@/chat/dto/like.response'; import { WebSocketExceptionFilter } from '@/middlewares/filter/webSocketException.filter'; import { StockService } from '@/stock/stock.service'; -import { LikeResponse } from '@/chat/dto/like.response'; -import { ChatScrollQuery, isChatScrollQuery } from '@/chat/dto/chat.request'; interface chatMessage { room: string; @@ -37,12 +40,16 @@ interface chatResponse { export class ChatGateway implements OnGatewayConnection { @WebSocketServer() server: Server; + websocketSessionService: WebsocketSessionService; constructor( @Inject('winston') private readonly logger: Logger, private readonly stockService: StockService, private readonly chatService: ChatService, - ) {} + @Inject(MEMORY_STORE) sessionStore: MemoryStore, + ) { + this.websocketSessionService = new WebsocketSessionService(sessionStore); + } @UseGuards(WebSocketSessionGuard) @SubscribeMessage('chat') @@ -74,13 +81,18 @@ export class ChatGateway implements OnGatewayConnection { async handleConnection(client: Socket) { try { + const user = + await this.websocketSessionService.getAuthenticatedUser(client); const { stockId, pageSize } = await this.getChatScrollQuery(client); await this.validateExistStock(stockId); client.join(stockId); - const messages = await this.chatService.scrollFirstChat({ - stockId, - pageSize, - }); + const messages = await this.chatService.scrollFirstChat( + { + stockId, + pageSize, + }, + user?.id, + ); this.logger.info(`client joined room ${stockId}`); client.emit('chat', messages); } catch (e) { From 74220b1c191169d8d0257078d3ec9f1cd64fc016 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 10:40:35 +0900 Subject: [PATCH 025/112] =?UTF-8?q?=E2=9C=A8=20feat:=20cors=20=ED=97=88?= =?UTF-8?q?=EC=9A=A9=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/main.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 0be16e68..935fea99 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -19,6 +19,10 @@ async function bootstrap() { transformOptions: { enableImplicitConversion: true }, }), ); + app.enableCors({ + origin: true, + credentials: true, + }); useSwagger(app); app.use(passport.initialize()); app.use(passport.session()); From c2f20d0518921e4553de973b18c695c80607e37b Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 10:41:52 +0900 Subject: [PATCH 026/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20?= =?UTF-8?q?=EC=9E=98=EB=AA=BB=EB=90=9C=20=ED=8C=8C=EC=9D=BC=EB=AA=85=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...sion.guard..ts => webSocketSession.guard.ts} | 0 .../auth/session/websocketSession.service.ts | 2 +- packages/backend/src/chat/chat.gateway.ts | 6 +++--- packages/backend/src/chat/chat.service.ts | 17 ++++++++--------- 4 files changed, 12 insertions(+), 13 deletions(-) rename packages/backend/src/auth/session/{webSocketSession.guard..ts => webSocketSession.guard.ts} (100%) diff --git a/packages/backend/src/auth/session/webSocketSession.guard..ts b/packages/backend/src/auth/session/webSocketSession.guard.ts similarity index 100% rename from packages/backend/src/auth/session/webSocketSession.guard..ts rename to packages/backend/src/auth/session/webSocketSession.guard.ts diff --git a/packages/backend/src/auth/session/websocketSession.service.ts b/packages/backend/src/auth/session/websocketSession.service.ts index edfd8ea9..10af73c2 100644 --- a/packages/backend/src/auth/session/websocketSession.service.ts +++ b/packages/backend/src/auth/session/websocketSession.service.ts @@ -1,7 +1,7 @@ import { MemoryStore } from 'express-session'; import { Socket } from 'socket.io'; import { websocketCookieParse } from '@/auth/session/cookieParser'; -import { PassportSession } from '@/auth/session/webSocketSession.guard.'; +import { PassportSession } from '@/auth/session/webSocketSession.guard'; export class WebsocketSessionService { constructor(private readonly sessionStore: MemoryStore) {} diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index 21ba2290..77db725d 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -13,7 +13,7 @@ import { Logger } from 'winston'; import { SessionSocket, WebSocketSessionGuard, -} from '@/auth/session/webSocketSession.guard.'; +} from '@/auth/session/webSocketSession.guard'; import { WebsocketSessionService } from '@/auth/session/websocketSession.service'; import { MEMORY_STORE } from '@/auth/session.module'; import { ChatService } from '@/chat/chat.service'; @@ -116,8 +116,8 @@ export class ChatGateway implements OnGatewayConnection { } return { stockId: query.stockId, - latestChatId: query.latestChatId, - pageSize: query.pageSize, + latestChatId: query.latestChatId ? Number(query.latestChatId) : undefined, + pageSize: query.pageSize ? Number(query.pageSize) : undefined, }; } diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 650c5fb4..569a44ef 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -24,21 +24,20 @@ export class ChatService { } async scrollFirstChat(chatScrollQuery: ChatScrollQuery, userId?: number) { - const { pageSize } = chatScrollQuery; - this.validatePageSize(pageSize); + this.validatePageSize(chatScrollQuery); const result = await this.findFirstChatScroll(chatScrollQuery, userId); - return await this.toScrollResponse(result, pageSize); + return await this.toScrollResponse(result, chatScrollQuery.pageSize); } async scrollNextChat(chatScrollQuery: ChatScrollQuery, userId?: number) { - const { pageSize } = chatScrollQuery; - this.validatePageSize(pageSize); + this.validatePageSize(chatScrollQuery); const result = await this.findChatScroll(chatScrollQuery, userId); - return await this.toScrollResponse(result, pageSize); + return await this.toScrollResponse(result, chatScrollQuery.pageSize); } - private validatePageSize(scrollSize?: number) { - if (scrollSize && scrollSize > 100) { + private validatePageSize(chatScrollQuery: ChatScrollQuery) { + const { pageSize } = chatScrollQuery; + if (pageSize && pageSize > 100) { throw new BadRequestException('pageSize should be less than 100'); } } @@ -100,7 +99,7 @@ export class ChatService { latestChatId, }) .orderBy('chat.id', 'DESC') - .limit(pageSize + 1) + .take(pageSize + 1) .getMany(); } } From 16e7592c16e302d41ce8521de12632ec6f06ed68 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 11:16:29 +0900 Subject: [PATCH 027/112] =?UTF-8?q?=E2=9C=85=20test:=20datasource=20mock?= =?UTF-8?q?=20=ED=83=80=EC=9E=85=20=EC=97=90=EB=9F=AC=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/user/user.service.spec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/user/user.service.spec.ts b/packages/backend/src/user/user.service.spec.ts index a0a3f19d..df80a82c 100644 --- a/packages/backend/src/user/user.service.spec.ts +++ b/packages/backend/src/user/user.service.spec.ts @@ -1,9 +1,9 @@ /* eslint-disable max-lines-per-function */ -import { BadRequestException, NotFoundException } from '@nestjs/common'; -import { DataSource, EntityManager } from 'typeorm'; -import { User } from './domain/user.entity'; -import { OauthType } from '@/user/domain/ouathType'; -import { UserService } from '@/user/user.service'; +import { BadRequestException, NotFoundException } from "@nestjs/common"; +import { DataSource, EntityManager } from "typeorm"; +import { User } from "./domain/user.entity"; +import { OauthType } from "@/user/domain/ouathType"; +import { UserService } from "@/user/user.service"; export function createDataSourceMock( managerMock?: Partial, @@ -15,7 +15,7 @@ export function createDataSourceMock( }; return { - getRepository: managerMock.getRepository, + getRepository: managerMock?.getRepository, transaction: jest.fn().mockImplementation(async (work) => { return work({ ...defaultManagerMock, ...managerMock }); }), From 5160dc5e151fc3915f27bbef9388325262e83f6f Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 11:16:56 +0900 Subject: [PATCH 028/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20?= =?UTF-8?q?=EC=B1=84=ED=8C=85=20=EC=8A=A4=ED=81=AC=EB=A1=A4=20=EC=A4=91?= =?UTF-8?q?=EB=B3=B5=20=EB=A1=9C=EC=A7=81=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 2 +- packages/backend/src/chat/chat.gateway.ts | 2 +- .../backend/src/chat/chat.service.spec.ts | 5 +- packages/backend/src/chat/chat.service.ts | 54 +++++-------------- 4 files changed, 17 insertions(+), 46 deletions(-) diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index d295f0c9..0670c9fa 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -53,7 +53,7 @@ export class ChatController { @Req() req: Express.Request, ) { const user = req.user as User; - return await this.chatService.scrollNextChat(request, user?.id); + return await this.chatService.scrollChat(request, user?.id); } @UseGuards(SessionGuard) diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index 77db725d..65474edb 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -86,7 +86,7 @@ export class ChatGateway implements OnGatewayConnection { const { stockId, pageSize } = await this.getChatScrollQuery(client); await this.validateExistStock(stockId); client.join(stockId); - const messages = await this.chatService.scrollFirstChat( + const messages = await this.chatService.scrollChat( { stockId, pageSize, diff --git a/packages/backend/src/chat/chat.service.spec.ts b/packages/backend/src/chat/chat.service.spec.ts index 24c77fae..1057ffed 100644 --- a/packages/backend/src/chat/chat.service.spec.ts +++ b/packages/backend/src/chat/chat.service.spec.ts @@ -8,9 +8,8 @@ describe('ChatService 테스트', () => { const chatService = new ChatService(dataSource as DataSource); await expect(() => - chatService.scrollNextChat({ + chatService.scrollChat({ stockId: 'A005930', - latestChatId: 1, pageSize: 101, }), ).rejects.toThrow('pageSize should be less than 100'); @@ -21,7 +20,7 @@ describe('ChatService 테스트', () => { const chatService = new ChatService(dataSource as DataSource); await expect(() => - chatService.scrollFirstChat({ stockId: 'A005930', pageSize: 101 }), + chatService.scrollChat({ stockId: 'A005930', pageSize: 101 }), ).rejects.toThrow('pageSize should be less than 100'); }); }); diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 569a44ef..3b9ae9d3 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -23,13 +23,7 @@ export class ChatService { }); } - async scrollFirstChat(chatScrollQuery: ChatScrollQuery, userId?: number) { - this.validatePageSize(chatScrollQuery); - const result = await this.findFirstChatScroll(chatScrollQuery, userId); - return await this.toScrollResponse(result, chatScrollQuery.pageSize); - } - - async scrollNextChat(chatScrollQuery: ChatScrollQuery, userId?: number) { + async scrollChat(chatScrollQuery: ChatScrollQuery, userId?: number) { this.validatePageSize(chatScrollQuery); const result = await this.findChatScroll(chatScrollQuery, userId); return await this.toScrollResponse(result, chatScrollQuery.pageSize); @@ -55,51 +49,29 @@ export class ChatService { chatScrollQuery: ChatScrollQuery, userId?: number, ) { - if (!chatScrollQuery.latestChatId) { - return await this.findFirstChatScroll(chatScrollQuery, userId); - } else { - return await this.findNextChatScroll(chatScrollQuery); - } + const queryBuilder = this.buildChatScrollQuery(chatScrollQuery, userId); + return queryBuilder.getMany(); } - private async findFirstChatScroll( + private buildChatScrollQuery( chatScrollQuery: ChatScrollQuery, userId?: number, ) { const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - if (!chatScrollQuery.pageSize) { - chatScrollQuery.pageSize = DEFAULT_PAGE_SIZE; - } - const { stockId, pageSize } = chatScrollQuery; - return queryBuilder + const { stockId, latestChatId, pageSize } = chatScrollQuery; + const size = pageSize ? pageSize : DEFAULT_PAGE_SIZE; + + queryBuilder .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { userId, }) .where('chat.stock_id = :stockId', { stockId }) .orderBy('chat.id', 'DESC') - .take(pageSize + 1) - .getMany(); - } - - private async findNextChatScroll( - chatScrollQuery: ChatScrollQuery, - userId?: number, - ) { - const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - if (!chatScrollQuery.pageSize) { - chatScrollQuery.pageSize = DEFAULT_PAGE_SIZE; + .take(size + 1); + if (latestChatId) { + queryBuilder.andWhere('chat.id < :latestChatId', { latestChatId }); } - const { stockId, latestChatId, pageSize } = chatScrollQuery; - return queryBuilder - .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { - userId, - }) - .where('chat.stock_id = :stockId and chat.id < :latestChatId', { - stockId, - latestChatId, - }) - .orderBy('chat.id', 'DESC') - .take(pageSize + 1) - .getMany(); + + return queryBuilder; } } From 6a56cc128fcf59e1aff88b7b3c31c80535b13dfb Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 11:26:27 +0900 Subject: [PATCH 029/112] =?UTF-8?q?=E2=9C=A8=20feat:=20detail=20=EC=99=84?= =?UTF-8?q?=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 132 ++++++++++++------ .../openapi/api/openapiMinuteData.api.ts | 2 +- .../openapi/api/openapiPeriodData.api.ts | 4 +- .../scraper/openapi/openapi-scraper.module.ts | 26 +++- .../openapi/type/openapiDetailData.type.ts | 64 ++++----- test.js | 8 -- yarn.lock | 65 ++++----- 7 files changed, 174 insertions(+), 127 deletions(-) delete mode 100644 test.js diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 77fa6feb..fb07ec1d 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -1,38 +1,36 @@ import { Cron } from '@nestjs/schedule'; -import { DataSource } from 'typeorm'; +import { Between, DataSource } from 'typeorm'; import { openApiConfig } from '../config/openapi.config'; import { getOpenApi } from '../openapiUtil.api'; import { DetailDataQuery, FinancialData, - FinancialDetail, isFinancialData, - isFinancialDetail, isProductDetail, ProductDetail, StockDetailQuery, } from '../type/openapiDetailData.type'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; -import { StockDetail } from '@/stock/domain/stockDetail.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; export class OpenapiDetailData { private readonly financialUrl: string = '/uapi/domestic-stock/v1/finance/financial-ratio'; private readonly defaultUrl: string = '/uapi/domestic-stock/v1/quotations/search-stock-info'; - private readonly incomeUrl: string = '/uapi/domestic-stock/v1/finance/income-statement'; - private readonly intervals = 4000; + private readonly incomeUrl: string = + '/uapi/domestic-stock/v1/finance/income-statement'; + private readonly intervals = 100; private readonly config: (typeof openApiConfig)[] = openApiToken.configs; constructor(private readonly datasource: DataSource) {} - @Cron('10 1 * * 1-5') + @Cron('0 8 * * 1-5') public async getDetailData() { const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); const configCount = this.config.length; - const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { @@ -41,47 +39,104 @@ export class OpenapiDetailData { } } - private async saveDetailData(output1: FinancialData, output2: ProductDetail, output3 : StockDaily[]) { + private async saveDetailData(stockDetail: StockDetail) { const entityManager = this.datasource.manager; const entity = StockDetail; - entityManager.create(entity, output1); + entityManager.create(entity, stockDetail); + } + + private async calPer(eps: number): Promise { + if (eps <= 0) return NaN; + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + const currentPrice = latestResult[0].close; + const per = currentPrice / eps; + + return per; + } + + private async calMarketCap(lstg: number) { + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + const currentPrice = latestResult[0].close; + const marketCap = lstg * currentPrice; + return marketCap; } - private makeStockDetailObject( - output1: FinancialDetail, + private async get52WeeksLowHigh() { + const manager = this.datasource.manager; + const nowDate = new Date(); + const weeksAgoDate = this.getDate52WeeksAgo(); + // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 + const output = await manager.find(StockDaily, { + select: ['low', 'high'], + where: { + startTime: Between(weeksAgoDate, nowDate), + }, + }); + const result = output.reduce((prev, cur) => { + if (prev.low > cur.low) prev.low = cur.low; + if (prev.high < cur.high) prev.high = cur.high; + return cur; + }, new StockDaily()); + return { low: result.low, high: result.high }; + } + + private async makeStockDetailObject( + output1: FinancialData, output2: ProductDetail, - ): StockDetail { + ): Promise { const result = new StockDetail(); - result.marketCap = output2. + result.marketCap = + (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; + result.eps = parseInt(output1.eps); + const { low, high } = await this.get52WeeksLowHigh(); + result.low52w = low; + result.high52w = high; + result.eps = parseInt(output1.eps); + result.per = await this.calPer(parseInt(output1.eps)); + result.updatedAt = new Date(); return result; } + private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { + const dataQuery = this.getDetailDataQuery(stock.id!); + const defaultQuery = this.getDefaultDataQuery(stock.id!); + + // 여기서 가져올 건 eps -> eps와 per 계산하자. + const output1 = await getOpenApi( + this.incomeUrl, + conf, + dataQuery, + 'FHKST66430200', + ); + // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 + const output2 = await getOpenApi( + this.defaultUrl, + conf, + defaultQuery, + 'CTPF1002R', + ); + + if (isFinancialData(output1) && isProductDetail(output2)) { + const stockDetail = await this.makeStockDetailObject(output1, output2); + this.saveDetailData(stockDetail); + } + } + private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { - const manager = this.datasource.manager; + let delay = 0; for (const stock of chunk) { - const dataQuery = this.getDetailDataQuery(stock.id!); - const defaultQuery = this.getDefaultDataQuery(stock.id!); - // 여기서 가져올 건 eps -> eps와 per 계산하자. - const output1 = await getOpenApi(this.incomeUrl, conf, dataQuery, 'FHKST66430200'); - // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 - const output2 = await getOpenApi( - this.defaultUrl, - conf, - defaultQuery, - 'CTPF1002R', - ); - // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 - const output3 = await manager.find(StockDaily, { - select: { - - }, - where: { - - } - }) - // 주식 마지막 데이터 끌고 오기. 최신 데이터로. - if ( isProductDetail(output1)) { - } + setTimeout(() => this.getDetailDataDelay(stock, conf), delay); + delay += this.intervals; } } @@ -114,5 +169,4 @@ export class OpenapiDetailData { date52WeeksAgo.setHours(0, 0, 0, 0); return date52WeeksAgo; } - } diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index cbc43334..e1532afa 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -1,3 +1,4 @@ +import { Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DataSource } from 'typeorm'; import { openApiConfig } from '../config/openapi.config'; @@ -10,7 +11,6 @@ import { import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; -import { Injectable } from '@nestjs/common'; @Injectable() export class OpenapiMinuteData { diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 3eac067f..5af605b1 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -1,3 +1,4 @@ +import { Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DataSource, EntityManager } from 'typeorm'; import { getOpenApi, getPreviousDate, getTodayDate } from '../openapiUtil.api'; @@ -16,7 +17,6 @@ import { StockMonthly, StockYearly, } from '@/stock/domain/stockData.entity'; -import { Injectable } from '@nestjs/common'; const DATE_TO_ENTITY = { D: StockDaily, @@ -38,7 +38,7 @@ const INTERVALS = 4000; export class OpenapiPeriodData { private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; - public constructor(private readonly datasourse: DataSource) { + public constructor(private readonly datasource: DataSource) { this.getItemChartPriceCheck(); } diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 00dca421..3e79d214 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,16 +1,32 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiScraperService } from './openapi-scraper.service'; -import { DataSource } from 'typeorm'; -import { TypeOrmModule } from '@nestjs/typeorm'; import { Stock } from '@/stock/domain/stock.entity'; -import { StockDaily, StockMinutely, StockMonthly, StockWeekly, StockYearly } from '@/stock/domain/stockData.entity'; -import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; +import { + StockDaily, + StockMinutely, + StockMonthly, + StockWeekly, + StockYearly, +} from '@/stock/domain/stockData.entity'; import { StockDetail } from '@/stock/domain/stockDetail.entity'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @Module({ - imports: [TypeOrmModule.forFeature([Stock, StockMinutely , StockDaily, StockWeekly, StockMonthly, StockYearly, StockLiveData, StockDetail])], + imports: [ + TypeOrmModule.forFeature([ + Stock, + StockMinutely, + StockDaily, + StockWeekly, + StockMonthly, + StockYearly, + StockLiveData, + StockDetail, + ]), + ], controllers: [], providers: [OpenapiPeriodData, OpenapiMinuteData, OpenapiScraperService], }) diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts index c5642b3a..f05edcea 100644 --- a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -178,36 +178,36 @@ export type StockDetailQuery = { code: string; }; -export type FinancialDetail = { - stac_yymm: string; // 결산 년월 - sale_account: string; // 매출액 - sale_cost: string; // 매출원가 - sale_totl_prfi: string; // 매출총이익 - depr_cost: string; // 감가상각비 - sell_mang: string; // 판매관리비 - bsop_prti: string; // 영업이익 - bsop_non_ernn: string; // 영업외수익 - bsop_non_expn: string; // 영업외비용 - op_prfi: string; // 영업이익 - spec_prfi: string; // 특별이익 - spec_loss: string; // 특별손실 - thtr_ntin: string; // 세전순이익 -}; +//export type FinancialDetail = { +// stac_yymm: string; // 결산 년월 +// sale_account: string; // 매출액 +// sale_cost: string; // 매출원가 +// sale_totl_prfi: string; // 매출총이익 +// depr_cost: string; // 감가상각비 +// sell_mang: string; // 판매관리비 +// bsop_prti: string; // 영업이익 +// bsop_non_ernn: string; // 영업외수익 +// bsop_non_expn: string; // 영업외비용 +// op_prfi: string; // 영업이익 +// spec_prfi: string; // 특별이익 +// spec_loss: string; // 특별손실 +// thtr_ntin: string; // 세전순이익 +//}; -export const isFinancialDetail = (data: any): data is FinancialDetail => { - return ( - typeof data.stac_yymm === 'string' && - typeof data.sale_account === 'string' && - typeof data.sale_cost === 'string' && - typeof data.sale_totl_prfi === 'string' && - typeof data.depr_cost === 'string' && - typeof data.sell_mang === 'string' && - typeof data.bsop_prti === 'string' && - typeof data.bsop_non_ernn === 'string' && - typeof data.bsop_non_expn === 'string' && - typeof data.op_prfi === 'string' && - typeof data.spec_prfi === 'string' && - typeof data.spec_loss === 'string' && - typeof data.thtr_ntin === 'string' - ); -}; +//export const isFinancialDetail = (data: any): data is FinancialDetail => { +// return ( +// typeof data.stac_yymm === 'string' && +// typeof data.sale_account === 'string' && +// typeof data.sale_cost === 'string' && +// typeof data.sale_totl_prfi === 'string' && +// typeof data.depr_cost === 'string' && +// typeof data.sell_mang === 'string' && +// typeof data.bsop_prti === 'string' && +// typeof data.bsop_non_ernn === 'string' && +// typeof data.bsop_non_expn === 'string' && +// typeof data.op_prfi === 'string' && +// typeof data.spec_prfi === 'string' && +// typeof data.spec_loss === 'string' && +// typeof data.thtr_ntin === 'string' +// ); +//}; diff --git a/test.js b/test.js deleted file mode 100644 index ea3c8120..00000000 --- a/test.js +++ /dev/null @@ -1,8 +0,0 @@ -function getDate52WeeksAgo() { - const today = new Date(); - const weeksAgo = 52 * 7; // 52주 * 7일 - const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); - return date52WeeksAgo; - } - - console.log(getDate52WeeksAgo()); diff --git a/yarn.lock b/yarn.lock index 10d4e991..b8b27206 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1199,11 +1199,6 @@ resolved "https://registry.yarnpkg.com/@lukeed/csprng/-/csprng-1.1.0.tgz#1e3e4bd05c1cc7a0b2ddbd8a03f39f6e4b5e6cfe" integrity sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA== -"@microsoft/tsdoc@^0.15.0": - version "0.15.0" - resolved "https://registry.yarnpkg.com/@microsoft/tsdoc/-/tsdoc-0.15.0.tgz#f29a55df17cb6e87cfbabce33ff6a14a9f85076d" - integrity sha512-HZpPoABogPvjeJOdzCOSJsXeL/SMCBgBZMVC3X3d7YYp2gf31MfxhUoYUNwf1ERPJOnQc0wkFn9trqI6ZEdZuA== - "@mdx-js/react@^3.0.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-3.1.0.tgz#c4522e335b3897b9a845db1dbdd2f966ae8fb0ed" @@ -1211,6 +1206,11 @@ dependencies: "@types/mdx" "^2.0.0" +"@microsoft/tsdoc@^0.15.0": + version "0.15.0" + resolved "https://registry.yarnpkg.com/@microsoft/tsdoc/-/tsdoc-0.15.0.tgz#f29a55df17cb6e87cfbabce33ff6a14a9f85076d" + integrity sha512-HZpPoABogPvjeJOdzCOSJsXeL/SMCBgBZMVC3X3d7YYp2gf31MfxhUoYUNwf1ERPJOnQc0wkFn9trqI6ZEdZuA== + "@nestjs/cli@^10.0.0": version "10.4.5" resolved "https://registry.yarnpkg.com/@nestjs/cli/-/cli-10.4.5.tgz#d6563b87e8ca1d0f256c19a7847dbcc96c76a88e" @@ -2000,7 +2000,6 @@ dependencies: "@types/node" "*" -"@types/estree@1.0.6", "@types/estree@^1.0.5", "@types/estree@^1.0.6": "@types/doctrine@^0.0.9": version "0.0.9" resolved "https://registry.yarnpkg.com/@types/doctrine/-/doctrine-0.0.9.tgz#d86a5f452a15e3e3113b99e39616a9baa0f9863f" @@ -2260,16 +2259,16 @@ dependencies: "@types/node" "*" -"@types/validator@^13.11.8": - version "13.12.2" - resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.12.2.tgz#760329e756e18a4aab82fc502b51ebdfebbe49f5" - integrity sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA== - "@types/uuid@^9.0.1": version "9.0.8" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA== +"@types/validator@^13.11.8": + version "13.12.2" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.12.2.tgz#760329e756e18a4aab82fc502b51ebdfebbe49f5" + integrity sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA== + "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -3022,13 +3021,6 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -better-opn@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-3.0.2.tgz#f96f35deaaf8f34144a4102651babcf00d1d8817" - integrity sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ== - dependencies: - open "^8.0.4" - base64id@2.0.0, base64id@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" @@ -3039,6 +3031,13 @@ base64url@3.x.x: resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== +better-opn@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-3.0.2.tgz#f96f35deaaf8f34144a4102651babcf00d1d8817" + integrity sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ== + dependencies: + open "^8.0.4" + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -6151,13 +6150,6 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== - dependencies: - argparse "^2.0.1" - jsdoc-type-pratt-parser@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-4.1.0.tgz#ff6b4a3f339c34a6c188cbf50a16087858d22113" @@ -6734,6 +6726,13 @@ neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== +nest-winston@^1.9.7: + version "1.9.7" + resolved "https://registry.yarnpkg.com/nest-winston/-/nest-winston-1.9.7.tgz#1ef6eb2459ce595655de37d5beb900d2e75b61d3" + integrity sha512-pTTgImRgv7urojsDvaTlenAjyJNLj7ywamfjzrhWKhLhp80AKLYNwf103dVHeqZWe+nzp/vd9DGRs/UN/YadOQ== + dependencies: + fast-safe-stringify "^2.1.1" + no-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d" @@ -6742,13 +6741,6 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" -nest-winston@^1.9.7: - version "1.9.7" - resolved "https://registry.yarnpkg.com/nest-winston/-/nest-winston-1.9.7.tgz#1ef6eb2459ce595655de37d5beb900d2e75b61d3" - integrity sha512-pTTgImRgv7urojsDvaTlenAjyJNLj7ywamfjzrhWKhLhp80AKLYNwf103dVHeqZWe+nzp/vd9DGRs/UN/YadOQ== - dependencies: - fast-safe-stringify "^2.1.1" - node-abort-controller@^3.0.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" @@ -7485,7 +7477,6 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" -reflect-metadata@^0.2.0: reflect-metadata@^0.2.0, reflect-metadata@^0.2.1: version "0.2.2" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.2.tgz#400c845b6cba87a21f2c65c4aeb158f4fa4d9c5b" @@ -8705,7 +8696,6 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== -utils-merge@1.0.1, utils-merge@1.x.x, utils-merge@^1.0.1: util@^0.12.5: version "0.12.5" resolved "https://registry.yarnpkg.com/util/-/util-0.12.5.tgz#5f17a6059b73db61a875668781a1c2b136bd6fbc" @@ -8717,16 +8707,11 @@ util@^0.12.5: is-typed-array "^1.1.3" which-typed-array "^1.1.2" -utils-merge@1.0.1: +utils-merge@1.0.1, utils-merge@1.x.x, utils-merge@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@^9.0.0: - version "9.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" - integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== - uuid@10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" From 8f5f7729fd612a129774320378c2be3bf2d6fb98 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 11:38:37 +0900 Subject: [PATCH 030/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=88=9C=EC=9C=BC=EB=A1=9C=20=EC=B1=84=ED=8C=85=20?= =?UTF-8?q?=EC=A0=95=EB=A0=AC=20=EA=B8=B0=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.service.ts | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 3b9ae9d3..95412659 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -29,6 +29,26 @@ export class ChatService { return await this.toScrollResponse(result, chatScrollQuery.pageSize); } + async scrollChatByLike(chatScrollQuery: ChatScrollQuery, userId?: number) { + this.validatePageSize(chatScrollQuery); + const result = await this.findChatScrollOrderByLike( + chatScrollQuery, + userId, + ); + return await this.toScrollResponse(result, chatScrollQuery.pageSize); + } + + async findChatScrollOrderByLike( + chatScrollQuery: ChatScrollQuery, + userId?: number, + ) { + const queryBuilder = await this.buildChatScrollByLikeQuery( + chatScrollQuery, + userId, + ); + return queryBuilder.getMany(); + } + private validatePageSize(chatScrollQuery: ChatScrollQuery) { const { pageSize } = chatScrollQuery; if (pageSize && pageSize > 100) { @@ -53,6 +73,41 @@ export class ChatService { return queryBuilder.getMany(); } + private async buildChatScrollByLikeQuery( + chatScrollQuery: ChatScrollQuery, + userId?: number, + ) { + const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); + const { stockId, latestChatId, pageSize } = chatScrollQuery; + const size = pageSize ? pageSize : DEFAULT_PAGE_SIZE; + + queryBuilder + .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { + userId, + }) + .where('chat.stock_id = :stockId', { stockId }) + .orderBy('chat.likeCount', 'DESC') + .addOrderBy('chat.id', 'DESC') + .take(size + 1); + if (latestChatId) { + const chat = await this.dataSource.manager.findOne(Chat, { + where: { id: latestChatId }, + select: ['likeCount'], + }); + if (chat) { + queryBuilder.andWhere( + 'chat.likeCount < :likeCount or (chat.likeCount = :likeCount and chat.id < :latestChatId)', + { + likeCount: chat.likeCount, + latestChatId, + }, + ); + } + } + + return queryBuilder; + } + private buildChatScrollQuery( chatScrollQuery: ChatScrollQuery, userId?: number, From cdcaf84a5b20446a82972053b47fd374f9066dfd Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 11:39:11 +0900 Subject: [PATCH 031/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A2=8B=EC=95=84?= =?UTF-8?q?=EC=9A=94=20=EC=88=9C=20=EC=B1=84=ED=8C=85=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A1=A4=20=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.controller.ts | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/backend/src/chat/chat.controller.ts b/packages/backend/src/chat/chat.controller.ts index 0670c9fa..35ce3a3b 100644 --- a/packages/backend/src/chat/chat.controller.ts +++ b/packages/backend/src/chat/chat.controller.ts @@ -64,4 +64,29 @@ export class ChatController { this.chatGateWay.broadcastLike(result); return result; } + + @ApiOperation({ + summary: '채팅 스크롤 조회 API(좋아요 순)', + description: '좋아요 순으로 채팅을 스크롤하여 조회한다.', + }) + @ApiOkResponse({ + description: '스크롤 조회 성공', + type: ChatScrollResponse, + }) + @ApiBadRequestResponse({ + description: '스크롤 크기 100 초과', + example: { + message: 'pageSize should be less than 100', + error: 'Bad Request', + statusCode: 400, + }, + }) + @Get('/like') + async findChatListByLike( + @Query() request: ChatScrollQuery, + @Req() req: Express.Request, + ) { + const user = req.user as User; + return await this.chatService.scrollChatByLike(request, user?.id); + } } From 8e27f1307ddbe1f951776ef7d6c8baba82ba0580 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 11:41:08 +0900 Subject: [PATCH 032/112] =?UTF-8?q?=E2=9C=A8=20feat:=20chat=20likeCount=20?= =?UTF-8?q?=EC=9D=B8=EB=8D=B1=EC=8A=A4=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/domain/chat.entity.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/chat/domain/chat.entity.ts b/packages/backend/src/chat/domain/chat.entity.ts index a1bab9ca..2a5ab380 100644 --- a/packages/backend/src/chat/domain/chat.entity.ts +++ b/packages/backend/src/chat/domain/chat.entity.ts @@ -1,6 +1,7 @@ import { Column, Entity, + Index, JoinColumn, ManyToOne, OneToMany, @@ -34,6 +35,7 @@ export class Chat { @Column({ type: 'enum', enum: ChatType, default: ChatType.NORMAL }) type: ChatType = ChatType.NORMAL; + @Index() @Column({ name: 'like_count', default: 0 }) likeCount: number = 0; From 0dd1b70f9b90a9b32bd3bb4313bcfc12e0f6af19 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 12:02:56 +0900 Subject: [PATCH 033/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20?= =?UTF-8?q?=EC=B1=84=ED=8C=85=20=EC=8A=A4=ED=81=AC=EB=A1=A4=20=EC=BF=BC?= =?UTF-8?q?=EB=A6=AC=20=EB=B9=8C=EB=8D=94=20=EC=A4=91=EB=B3=B5=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.service.ts | 67 +++++++++++++---------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index 95412659..c9431a8d 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -1,5 +1,5 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, SelectQueryBuilder } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; import { ChatScrollQuery } from '@/chat/dto/chat.request'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; @@ -9,6 +9,13 @@ export interface ChatMessage { stockId: string; } +const ORDER = { + LIKE: 'like', + LATEST: 'latest', +} as const; + +export type Order = (typeof ORDER)[keyof typeof ORDER]; + const DEFAULT_PAGE_SIZE = 20; @Injectable() @@ -42,9 +49,10 @@ export class ChatService { chatScrollQuery: ChatScrollQuery, userId?: number, ) { - const queryBuilder = await this.buildChatScrollByLikeQuery( + const queryBuilder = await this.buildChatScrollQuery( chatScrollQuery, userId, + ORDER.LIKE, ); return queryBuilder.getMany(); } @@ -69,13 +77,17 @@ export class ChatService { chatScrollQuery: ChatScrollQuery, userId?: number, ) { - const queryBuilder = this.buildChatScrollQuery(chatScrollQuery, userId); + const queryBuilder = await this.buildChatScrollQuery( + chatScrollQuery, + userId, + ); return queryBuilder.getMany(); } - private async buildChatScrollByLikeQuery( + private async buildChatScrollQuery( chatScrollQuery: ChatScrollQuery, userId?: number, + order: Order = ORDER.LATEST, ) { const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); const { stockId, latestChatId, pageSize } = chatScrollQuery; @@ -86,9 +98,26 @@ export class ChatService { userId, }) .where('chat.stock_id = :stockId', { stockId }) - .orderBy('chat.likeCount', 'DESC') - .addOrderBy('chat.id', 'DESC') .take(size + 1); + + if (order === ORDER.LIKE) { + return this.buildLikeCountQuery(queryBuilder, latestChatId); + } + queryBuilder.orderBy('chat.id', 'DESC'); + if (latestChatId) { + queryBuilder.andWhere('chat.id < :latestChatId', { latestChatId }); + } + + return queryBuilder; + } + + private async buildLikeCountQuery( + queryBuilder: SelectQueryBuilder, + latestChatId?: number, + ) { + queryBuilder + .orderBy('chat.likeCount', 'DESC') + .addOrderBy('chat.id', 'DESC'); if (latestChatId) { const chat = await this.dataSource.manager.findOne(Chat, { where: { id: latestChatId }, @@ -96,7 +125,8 @@ export class ChatService { }); if (chat) { queryBuilder.andWhere( - 'chat.likeCount < :likeCount or (chat.likeCount = :likeCount and chat.id < :latestChatId)', + 'chat.likeCount < :likeCount or' + + ' (chat.likeCount = :likeCount and chat.id < :latestChatId)', { likeCount: chat.likeCount, latestChatId, @@ -104,29 +134,6 @@ export class ChatService { ); } } - - return queryBuilder; - } - - private buildChatScrollQuery( - chatScrollQuery: ChatScrollQuery, - userId?: number, - ) { - const queryBuilder = this.dataSource.createQueryBuilder(Chat, 'chat'); - const { stockId, latestChatId, pageSize } = chatScrollQuery; - const size = pageSize ? pageSize : DEFAULT_PAGE_SIZE; - - queryBuilder - .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { - userId, - }) - .where('chat.stock_id = :stockId', { stockId }) - .orderBy('chat.id', 'DESC') - .take(size + 1); - if (latestChatId) { - queryBuilder.andWhere('chat.id < :latestChatId', { latestChatId }); - } - return queryBuilder; } } From 5e7e4b09c3c9974828f4093ef8d53e6d865d8341 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 12:04:21 +0900 Subject: [PATCH 034/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A3=BC=EC=8B=9D?= =?UTF-8?q?=20=EC=86=8C=EC=9C=A0=20=ED=99=95=EC=9D=B8=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=EB=A5=BC=20=EC=BF=BC=EB=A6=AC?= =?UTF-8?q?=EB=A1=9C=20=EB=B0=9B=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/stock.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/stock/stock.controller.ts b/packages/backend/src/stock/stock.controller.ts index 5aaf6707..6f9e1dd4 100644 --- a/packages/backend/src/stock/stock.controller.ts +++ b/packages/backend/src/stock/stock.controller.ts @@ -140,7 +140,7 @@ export class StockController { }) @Get('user/ownership') async checkOwnership( - @Body() body: UserStockRequest, + @Query() body: UserStockRequest, @Req() request: Request, ) { const user = request.user as User; From ca6f156dec878a46375161aba90ebc04b5040ad3 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 12:39:36 +0900 Subject: [PATCH 035/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20tR=5FI?= =?UTF-8?q?DS=EB=A1=9C=20=EB=A6=AC=ED=84=B0=EB=9F=B4=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/scraper/openapi/type/openapiUtil.type.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts index df05e35a..bd2e3edd 100644 --- a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts @@ -1,7 +1,13 @@ export type TR_ID = | 'FHKST03010100' + | 'FHKST03010200' | 'FHKST66430200' | 'HHKDB669107C0' | 'CTPF1002R'; -export const DEFAULT_TR_ID: TR_ID = 'FHKST03010100'; +export const TR_IDS: Record = { + ITEM_CHART_PRICE: 'FHKST03010100', + MINUTE_DATA: 'FHKST03010200', + FINANCIAL_DATA: 'FHKST66430200', + PRODUCTION_DETAIL: 'CTPF1002R', +}; From 2429423cc9d7a46a95a0bf00cb05add5b76f2fe3 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 14:22:17 +0900 Subject: [PATCH 036/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20production=20?= =?UTF-8?q?=EB=AA=A8=EB=93=9C=EC=9D=BC=EB=95=8C=EC=97=90=EB=A7=8C=20?= =?UTF-8?q?=EC=9E=91=EB=8F=99=ED=95=98=EA=B2=8C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scraper/openapi/openapi-scraper.module.ts | 4 +--- .../scraper/openapi/openapi-scraper.service.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 3e79d214..9024497e 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,7 +1,5 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { OpenapiMinuteData } from './api/openapiMinuteData.api'; -import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiScraperService } from './openapi-scraper.service'; import { Stock } from '@/stock/domain/stock.entity'; import { @@ -28,6 +26,6 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; ]), ], controllers: [], - providers: [OpenapiPeriodData, OpenapiMinuteData, OpenapiScraperService], + providers: [OpenapiScraperService], }) export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts index 98f27a34..eb2b727b 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts @@ -1,13 +1,19 @@ import { Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; @Injectable() export class OpenapiScraperService { - public constructor( - private readonly datasourse: DataSource, - private readonly openapiPeriodData: OpenapiPeriodData, - private readonly openapiMinuteData: OpenapiMinuteData, - ) {} + private readonly openapiPeriodData: OpenapiPeriodData; + private readonly openapiMinuteData: OpenapiMinuteData; + private readonly openapiDetailData: OpenapiDetailData; + public constructor(private datasource: DataSource) { + if (process.env.NODE_ENV === 'production') { + this.openapiPeriodData = new OpenapiPeriodData(datasource); + this.openapiMinuteData = new OpenapiMinuteData(datasource); + this.openapiDetailData = new OpenapiDetailData(datasource); + } + } } From b7731690fd6c3a5ea7b25121facf8450b5f55dc1 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 14:22:45 +0900 Subject: [PATCH 037/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20?= =?UTF-8?q?=EB=A6=AC=ED=84=B0=EB=9F=B4=20=EC=BD=94=EB=93=9C=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0=20-=20tr=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../korea-stock-info/entities/stock.entity.ts | 23 -------- .../korea-stock-info.service.ts | 2 +- .../openapi/api/openapiDetailData.api.ts | 9 ++-- .../openapi/api/openapiMinuteData.api.ts | 53 ++++++++++++++----- .../openapi/api/openapiPeriodData.api.ts | 2 + .../src/scraper/openapi/openapiUtil.api.ts | 4 +- 6 files changed, 49 insertions(+), 44 deletions(-) delete mode 100644 packages/backend/src/scraper/korea-stock-info/entities/stock.entity.ts diff --git a/packages/backend/src/scraper/korea-stock-info/entities/stock.entity.ts b/packages/backend/src/scraper/korea-stock-info/entities/stock.entity.ts deleted file mode 100644 index ad721147..00000000 --- a/packages/backend/src/scraper/korea-stock-info/entities/stock.entity.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Entity, Column, PrimaryGeneratedColumn } from 'typeorm'; - -//TODO : entity update require -@Entity() -export class Master { - @PrimaryGeneratedColumn({ type: 'int', unsigned: true }) - id?: number; - - @Column() - shortCode?: string; - - @Column() - standardCode?: string; - - @Column() - koreanName?: string; - - @Column() - groupCode?: string; - - @Column() - marketCapSize?: string; -} diff --git a/packages/backend/src/scraper/korea-stock-info/korea-stock-info.service.ts b/packages/backend/src/scraper/korea-stock-info/korea-stock-info.service.ts index 618bb349..430b9f6a 100644 --- a/packages/backend/src/scraper/korea-stock-info/korea-stock-info.service.ts +++ b/packages/backend/src/scraper/korea-stock-info/korea-stock-info.service.ts @@ -21,7 +21,7 @@ export class KoreaStockInfoService { private readonly datasource: DataSource, @Inject('winston') private readonly logger: Logger, ) { - //this.initKoreaStockInfo(); + this.initKoreaStockInfo(); } private async existsStockInfo(stockId: string, manager: EntityManager) { diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index fb07ec1d..f8e3cb92 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -10,6 +10,7 @@ import { ProductDetail, StockDetailQuery, } from '../type/openapiDetailData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; @@ -40,9 +41,9 @@ export class OpenapiDetailData { } private async saveDetailData(stockDetail: StockDetail) { - const entityManager = this.datasource.manager; + const manager = this.datasource.manager; const entity = StockDetail; - entityManager.create(entity, stockDetail); + manager.save(entity, stockDetail); } private async calPer(eps: number): Promise { @@ -116,14 +117,14 @@ export class OpenapiDetailData { this.incomeUrl, conf, dataQuery, - 'FHKST66430200', + TR_IDS.FINANCIAL_DATA, ); // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 const output2 = await getOpenApi( this.defaultUrl, conf, defaultQuery, - 'CTPF1002R', + TR_IDS.PRODUCTION_DETAIL, ); if (isFinancialData(output1) && isProductDetail(output2)) { diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index e1532afa..26ea64e4 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -8,6 +8,7 @@ import { MinuteData, UpdateStockQuery, } from '../type/openapiMinuteData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; @@ -18,11 +19,14 @@ export class OpenapiMinuteData { private readonly entity = StockMinutely; private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; - public constructor(private readonly datasourse: DataSource) {} + private readonly intervals: number = 60; + public constructor(private readonly datasource: DataSource) { + this.getStockData().then(() => this.getMinuteData()); + } @Cron('0 1 * * 1-5') private async getStockData() { - this.stock = await this.datasourse.manager.findBy(Stock, { + this.stock = await this.datasource.manager.findBy(Stock, { isTrading: true, }); } @@ -44,28 +48,49 @@ export class OpenapiMinuteData { return stockPeriod; } - private async saveMinuteData(stockPeriod: StockMinutely) { - const manager = this.datasourse.manager; - manager.create(this.entity, stockPeriod); + private async saveMinuteData(stockId: string, item: MinuteData) { + const manager = this.datasource.manager; + const stockPeriod = this.convertResToMinuteData(stockId, item); + manager.save(this.entity, stockPeriod); + } + + private async getMinuteDataInterval( + stockId: string, + time: string, + config: typeof openApiConfig, + ) { + const query = this.getUpdateStockQuery(stockId, time); + console.log(query); + const response = await getOpenApi( + this.url, + config, + query, + TR_IDS.MINUTE_DATA, + ); + let output; + if (response.output2) output = response.output2; + if (output && output[0] && isMinuteData(output[0])) { + this.saveMinuteData(stockId, output[0]); + } } private async getMinuteDataChunk( chunk: Stock[], config: typeof openApiConfig, ) { + const time = getCurrentTime(); + let interval = 0; for await (const stock of chunk) { - const time = getCurrentTime(); - const query = this.getUpdateStockQuery(stock.id!, time); - const response = await getOpenApi(this.url, config, query); - const output = (await response.data).output2[0] as MinuteData; - if (output && isMinuteData(output)) { - const stockPeriod = this.convertResToMinuteData(stock.id!, output); - this.saveMinuteData(stockPeriod); - } + setTimeout( + () => this.getMinuteDataInterval(stock.id!, time, config), + interval, + ); + interval += this.intervals; } } - @Cron('* 9-16 * * 1-5') + @Cron('* 9-14 * * 1-5') + @Cron('0-30 15 * * 1-5') private getMinuteData() { const configCount = openApiToken.configs.length; const chunkSize = Math.ceil(this.stock.length / configCount); diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 5af605b1..c65b7789 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -8,6 +8,7 @@ import { ItemChartPriceQuery, Period, } from '../type/openapiPeriodData'; +import { TR_IDS } from '../type/openapiUtil.type'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { @@ -123,6 +124,7 @@ export class OpenapiPeriodData { this.url, openApiToken.configs[configIdx], query, + TR_IDS.ITEM_CHART_PRICE, ); return response.output2 as ChartData[]; } diff --git a/packages/backend/src/scraper/openapi/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/openapiUtil.api.ts index 06ae8c23..9ce04bd2 100644 --- a/packages/backend/src/scraper/openapi/openapiUtil.api.ts +++ b/packages/backend/src/scraper/openapi/openapiUtil.api.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import { openApiConfig } from './config/openapi.config'; -import { DEFAULT_TR_ID, TR_ID } from './type/openapiUtil.type'; +import { TR_ID } from './type/openapiUtil.type'; const postOpenApi = async ( url: string, @@ -19,7 +19,7 @@ const getOpenApi = async ( url: string, config: typeof openApiConfig, query: object, - tr_id: TR_ID = DEFAULT_TR_ID, + tr_id: TR_ID, ) => { try { const response = await axios.get(config.STOCK_URL + url, { From 8f30d02592da26024ce4da1a8bb3a5a23025038e Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 14:56:39 +0900 Subject: [PATCH 038/112] =?UTF-8?q?=E2=9C=A8=20feat:=20token=20retry=20?= =?UTF-8?q?=EB=A1=9C=EC=A7=81=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scraper/openapi/api/openapiToken.api.ts | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index c4df3cdf..e5c230cf 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,8 +1,9 @@ -import { Inject, NotFoundException } from '@nestjs/common'; +import { Inject } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; -import { postOpenApi } from '../openapiUtil.api'; +import { OpenapiException } from '../util/openapiCustom.error'; +import { postOpenApi } from '../util/openapiUtil.api'; import { logger } from '@/configs/logger.config'; class OpenapiTokenApi { @@ -34,8 +35,26 @@ class OpenapiTokenApi { } private async initAuthenValue() { - await this.initAccessToken(); - await this.initWebSocketKey(); + const delay = 60000; + const delayMinute = delay / 1000 / 60; + + try { + await this.initAccessToken(); + await this.initWebSocketKey(); + } catch (error) { + if (error instanceof Error) { + this.logger.warn( + `Request failed: ${error.message}. Retrying in ${delayMinute} minute...`, + ); + } else { + this.logger.warn( + `Request failed. Retrying in ${delayMinute} minute...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + await this.initAccessToken(); + await this.initWebSocketKey(); + } + } } @Cron('50 0 * * 1-5') @@ -64,7 +83,7 @@ class OpenapiTokenApi { }; const tmp = await postOpenApi('/oauth2/tokenP', config, body); if (!tmp.access_token) { - throw new NotFoundException('Access Token Failed'); + throw new OpenapiException('Access Token Failed', 403); } return tmp.access_token as string; } @@ -77,7 +96,7 @@ class OpenapiTokenApi { }; const tmp = await postOpenApi('/oauth2/Approval', config, body); if (!tmp.approval_key) { - throw new NotFoundException('WebSocket Key Failed'); + throw new OpenapiException('WebSocket Key Failed', 403); } return tmp.approval_key as string; } From 364f833bb9d7b1d3e0130df91657d5869d79030e Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 15:11:10 +0900 Subject: [PATCH 039/112] =?UTF-8?q?=F0=9F=92=84=20style:=20stock=20control?= =?UTF-8?q?ler=20import=20=EC=88=9C=EC=84=9C=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/stock.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/stock/stock.controller.ts b/packages/backend/src/stock/stock.controller.ts index 6f9e1dd4..3f6ef023 100644 --- a/packages/backend/src/stock/stock.controller.ts +++ b/packages/backend/src/stock/stock.controller.ts @@ -32,6 +32,7 @@ import { StockDetailService } from './stockDetail.service'; import SessionGuard from '@/auth/session/session.guard'; import { GetUser } from '@/common/decorator/user.decorator'; import { sessionConfig } from '@/configs/session.config'; +import { StockSearchRequest } from '@/stock/dto/stock.request'; import { StockSearchResponse, StockViewsResponse, @@ -46,7 +47,6 @@ import { UserStockResponse, } from '@/stock/dto/userStock.response'; import { User } from '@/user/domain/user.entity'; -import { StockSearchRequest } from '@/stock/dto/stock.request'; @Controller('stock') export class StockController { From b097178a9ff14c90c97ba19ce5b413ce17733f48 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 15:14:33 +0900 Subject: [PATCH 040/112] =?UTF-8?q?=E2=9C=A8=20feat:=20custom=20filter=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80,=20exception=EB=8F=84=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Decorator/openapiException.filter.ts | 32 +++++++++++++++++++ .../openapi/api/openapiDetailData.api.ts | 5 ++- .../openapi/api/openapiMinuteData.api.ts | 6 ++-- .../openapi/api/openapiPeriodData.api.ts | 10 ++++-- .../scraper/openapi/api/openapiToken.api.ts | 5 ++- .../openapi/util/openapiCustom.error.ts | 13 ++++++++ .../openapi/{ => util}/openapiUtil.api.ts | 27 +++++++++++++--- 7 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts create mode 100644 packages/backend/src/scraper/openapi/util/openapiCustom.error.ts rename packages/backend/src/scraper/openapi/{ => util}/openapiUtil.api.ts (67%) diff --git a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts new file mode 100644 index 00000000..50f91173 --- /dev/null +++ b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts @@ -0,0 +1,32 @@ +import { + ExceptionFilter, + Catch, + HttpException, + HttpStatus, + Logger, +} from '@nestjs/common'; +import { OpenapiException } from '../util/openapiCustom.error'; + +@Catch() +export class OpenapiExceptionFilter implements ExceptionFilter { + private readonly logger = new Logger(OpenapiExceptionFilter.name); + + catch(exception: unknown) { + const status = + exception instanceof HttpException + ? exception.getStatus() + : HttpStatus.INTERNAL_SERVER_ERROR; + + const message = + exception instanceof HttpException + ? exception.getResponse() + : 'Internal server error'; + + const error = + exception instanceof OpenapiException ? exception.getError() : ''; + + this.logger.error( + `HTTP Status: ${status} Error Message: ${JSON.stringify(message)} Error : ${error}`, + ); + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index f8e3cb92..a758df55 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -1,7 +1,8 @@ +import { UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Between, DataSource } from 'typeorm'; import { openApiConfig } from '../config/openapi.config'; -import { getOpenApi } from '../openapiUtil.api'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { DetailDataQuery, FinancialData, @@ -11,6 +12,7 @@ import { StockDetailQuery, } from '../type/openapiDetailData.type'; import { TR_IDS } from '../type/openapiUtil.type'; +import { getOpenApi } from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; @@ -28,6 +30,7 @@ export class OpenapiDetailData { constructor(private readonly datasource: DataSource) {} @Cron('0 8 * * 1-5') + @UseFilters(OpenapiExceptionFilter) public async getDetailData() { const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index 26ea64e4..60bc90b0 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -1,14 +1,15 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DataSource } from 'typeorm'; import { openApiConfig } from '../config/openapi.config'; -import { getCurrentTime, getOpenApi } from '../openapiUtil.api'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { isMinuteData, MinuteData, UpdateStockQuery, } from '../type/openapiMinuteData.type'; import { TR_IDS } from '../type/openapiUtil.type'; +import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; @@ -25,6 +26,7 @@ export class OpenapiMinuteData { } @Cron('0 1 * * 1-5') + @UseFilters(OpenapiExceptionFilter) private async getStockData() { this.stock = await this.datasource.manager.findBy(Stock, { isTrading: true, diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index c65b7789..fa4b2b9b 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -1,7 +1,7 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DataSource, EntityManager } from 'typeorm'; -import { getOpenApi, getPreviousDate, getTodayDate } from '../openapiUtil.api'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { ChartData, isChartData, @@ -9,6 +9,11 @@ import { Period, } from '../type/openapiPeriodData'; import { TR_IDS } from '../type/openapiUtil.type'; +import { + getOpenApi, + getPreviousDate, + getTodayDate, +} from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { @@ -44,6 +49,7 @@ export class OpenapiPeriodData { } @Cron('0 1 * * 1-5') + @UseFilters(OpenapiExceptionFilter) public async getItemChartPriceCheck() { const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index e5c230cf..36936c89 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,7 +1,8 @@ -import { Inject } from '@nestjs/common'; +import { Inject, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; import { logger } from '@/configs/logger.config'; @@ -58,6 +59,7 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') + @UseFilters(OpenapiExceptionFilter) private async initAccessToken() { const updatedConfig = await Promise.all( this.config.map(async (val) => { @@ -69,6 +71,7 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') + @UseFilters(OpenapiExceptionFilter) private async initWebSocketKey() { this.config.forEach(async (val) => { val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; diff --git a/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts b/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts new file mode 100644 index 00000000..1e0c3913 --- /dev/null +++ b/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts @@ -0,0 +1,13 @@ +import { HttpException, HttpStatus } from '@nestjs/common'; + +export class OpenapiException extends HttpException { + private error: unknown; + constructor(message: string, status: HttpStatus, error?: unknown) { + super(message, status); + this.error = error; + } + + public getError() { + return this.error; + } +} diff --git a/packages/backend/src/scraper/openapi/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts similarity index 67% rename from packages/backend/src/scraper/openapi/openapiUtil.api.ts rename to packages/backend/src/scraper/openapi/util/openapiUtil.api.ts index 9ce04bd2..7c177443 100644 --- a/packages/backend/src/scraper/openapi/openapiUtil.api.ts +++ b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts @@ -1,6 +1,25 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +import { HttpStatus } from '@nestjs/common'; import axios from 'axios'; -import { openApiConfig } from './config/openapi.config'; -import { TR_ID } from './type/openapiUtil.type'; +import { openApiConfig } from '../config/openapi.config'; +import { TR_ID } from '../type/openapiUtil.type'; +import { OpenapiException } from './openapiCustom.error'; + +const throwOpenapiException = (error: any) => { + if (error.message && error.response && error.response.status) { + throw new OpenapiException( + `Request failed: ${error.message}`, + error.response.status, + error, + ); + } else { + throw new OpenapiException( + `Unknown error: ${error.message || 'No message'}`, + HttpStatus.INTERNAL_SERVER_ERROR, + error, + ); + } +}; const postOpenApi = async ( url: string, @@ -11,7 +30,7 @@ const postOpenApi = async ( const response = await axios.post(config.STOCK_URL + url, body); return response.data; } catch (error) { - throw new Error(`Request failed: ${error}`); + throwOpenapiException(error); } }; @@ -34,7 +53,7 @@ const getOpenApi = async ( }); return response.data; } catch (error) { - throw new Error(`Request failed: ${error}`); + throwOpenapiException(error); } }; From adf6ac533a46baffc18d3db64711b92608cbe063 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 15:53:21 +0900 Subject: [PATCH 041/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A3=BC=EC=8B=9D?= =?UTF-8?q?=EC=9D=84=20=EC=86=8C=EC=9C=A0=ED=95=9C=20=EC=82=AC=EC=9A=A9?= =?UTF-8?q?=EC=9E=90=EB=A7=8C=20=EC=B1=84=ED=8C=85=20=EA=B0=80=EB=8A=A5?= =?UTF-8?q?=ED=95=98=EB=8F=84=EB=A1=9D=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.service.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index c9431a8d..fbb5ac45 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -1,8 +1,10 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { DataSource, SelectQueryBuilder } from 'typeorm'; +import { WsException } from '@nestjs/websockets'; +import { DataSource, EntityManager, SelectQueryBuilder } from 'typeorm'; import { Chat } from '@/chat/domain/chat.entity'; import { ChatScrollQuery } from '@/chat/dto/chat.request'; import { ChatScrollResponse } from '@/chat/dto/chat.response'; +import { UserStock } from '@/stock/domain/userStock.entity'; export interface ChatMessage { message: string; @@ -23,10 +25,15 @@ export class ChatService { constructor(private readonly dataSource: DataSource) {} async saveChat(userId: number, chatMessage: ChatMessage) { - return this.dataSource.manager.save(Chat, { - user: { id: userId }, - stock: { id: chatMessage.stockId }, - message: chatMessage.message, + return this.dataSource.transaction(async (manager) => { + if (!(await this.hasStock(userId, chatMessage.stockId, manager))) { + throw new WsException('not have stock'); + } + return manager.save(Chat, { + user: { id: userId }, + stock: { id: chatMessage.stockId }, + message: chatMessage.message, + }); }); } @@ -57,6 +64,12 @@ export class ChatService { return queryBuilder.getMany(); } + private hasStock(userId: number, stockId: string, manager: EntityManager) { + return manager.exists(UserStock, { + where: { user: { id: userId }, stock: { id: stockId } }, + }); + } + private validatePageSize(chatScrollQuery: ChatScrollQuery) { const { pageSize } = chatScrollQuery; if (pageSize && pageSize > 100) { From 9f28b3abe3d71c9fcffc68f2624d66efd70a12e7 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 18:22:21 +0900 Subject: [PATCH 042/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20minute=20fix=20pe?= =?UTF-8?q?r=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- output | 99 +++++++++++++++++++ .../Decorator/openapiException.filter.ts | 4 +- .../openapi/api/openapiDetailData.api.ts | 1 + .../openapi/api/openapiMinuteData.api.ts | 67 +++++++++---- .../openapi/api/openapiPeriodData.api.ts | 3 +- .../scraper/openapi/api/openapiToken.api.ts | 5 +- .../scraper/openapi/openapi-scraper.module.ts | 10 +- .../openapi/openapi-scraper.service.ts | 18 ++-- 8 files changed, 168 insertions(+), 39 deletions(-) create mode 100644 output diff --git a/output b/output new file mode 100644 index 00000000..cdd2d49a --- /dev/null +++ b/output @@ -0,0 +1,99 @@ +yarn workspace v1.22.22 +yarn run v1.22.22 +$ nest start --watch +[5:32:54 PM] Starting compilation in watch mode... + +[5:32:56 PM] Found 0 errors. Watching for file changes. + +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [NestFactory] Starting Nest application... +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] AppModule dependencies initialized +41ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ScraperModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] SessionModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ConfigHostModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] WinstonModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] DiscoveryModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ConfigModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ScheduleModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmCoreModule dependencies initialized +57ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] KoreaStockInfoModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] OpenapiScraperModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] UserModule dependencies initialized +1ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] StockModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ChatModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] AuthModule dependencies initialized +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [WebSocketsController] StockGateway subscribed to the "connectStock" message +13ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [WebSocketsController] ChatGateway subscribed to the "chat" message +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] StockController {/api/stock}: +1ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/view, POST} route +1ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user, POST} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user, DELETE} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user/ownership, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/minutely, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/daily, GET} route +1ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/weekly, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/mothly, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/yearly, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/detail, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topViews, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topGainers, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topLosers, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] UserController {/api/user}: +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, PATCH} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] GoogleAuthController {/api/auth/google}: +1ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/login, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/redirect, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/status, GET} route +0ms +[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [NestApplication] Nest application successfully started +5ms +[5:34:34 PM] File change detected. Starting incremental compilation... + +[5:34:34 PM] Found 0 errors. Watching for file changes. + +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [NestFactory] Starting Nest application... +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] AppModule dependencies initialized +40ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ScraperModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] SessionModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ConfigHostModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] WinstonModule dependencies initialized +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] DiscoveryModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ConfigModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ScheduleModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmCoreModule dependencies initialized +56ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] KoreaStockInfoModule dependencies initialized +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] OpenapiScraperModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] UserModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] StockModule dependencies initialized +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ChatModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] AuthModule dependencies initialized +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [WebSocketsController] StockGateway subscribed to the "connectStock" message +11ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [WebSocketsController] ChatGateway subscribed to the "chat" message +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] StockController {/api/stock}: +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/view, POST} route +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user, POST} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user, DELETE} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user/ownership, GET} route +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/minutely, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/daily, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/weekly, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/mothly, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/yearly, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/detail, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topViews, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topGainers, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topLosers, GET} route +1ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] UserController {/api/user}: +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, PATCH} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] GoogleAuthController {/api/auth/google}: +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/login, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/redirect, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/status, GET} route +0ms +[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [NestApplication] Nest application successfully started +5ms diff --git a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts index 50f91173..e86915b3 100644 --- a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts +++ b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts @@ -3,13 +3,13 @@ import { Catch, HttpException, HttpStatus, - Logger, } from '@nestjs/common'; +import { Logger } from 'winston'; import { OpenapiException } from '../util/openapiCustom.error'; @Catch() export class OpenapiExceptionFilter implements ExceptionFilter { - private readonly logger = new Logger(OpenapiExceptionFilter.name); + private readonly logger = new Logger(); catch(exception: unknown) { const status = diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index a758df55..1745d0d6 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -32,6 +32,7 @@ export class OpenapiDetailData { @Cron('0 8 * * 1-5') @UseFilters(OpenapiExceptionFilter) public async getDetailData() { + if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); const configCount = this.config.length; diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index 60bc90b0..e055de6e 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -14,23 +14,37 @@ import { openApiToken } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; +const STOCK_CUT = 4; + @Injectable() export class OpenapiMinuteData { - private stock: Stock[]; + private stock: Stock[][] = []; private readonly entity = StockMinutely; private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; - private readonly intervals: number = 60; + private readonly intervals: number = 130; + private flip: number = 0; public constructor(private readonly datasource: DataSource) { - this.getStockData().then(() => this.getMinuteData()); + this.getStockData(); } @Cron('0 1 * * 1-5') @UseFilters(OpenapiExceptionFilter) private async getStockData() { - this.stock = await this.datasource.manager.findBy(Stock, { + if (process.env.NODE_ENV !== 'production') return; + const stock = await this.datasource.manager.findBy(Stock, { isTrading: true, }); + const stockSize = Math.ceil(stock.length / STOCK_CUT); + let i = 0; + this.stock = []; + while (i < STOCK_CUT) { + this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); + i++; + } + console.log(stock.length); + console.log(this.stock.length); + console.log(this.stock[0].length); } private convertResToMinuteData(stockId: string, item: MinuteData) { @@ -50,9 +64,11 @@ export class OpenapiMinuteData { return stockPeriod; } - private async saveMinuteData(stockId: string, item: MinuteData) { + private async saveMinuteData(stockId: string, item: MinuteData[]) { const manager = this.datasource.manager; - const stockPeriod = this.convertResToMinuteData(stockId, item); + const stockPeriod = item.map((val) => + this.convertResToMinuteData(stockId, val), + ); manager.save(this.entity, stockPeriod); } @@ -63,19 +79,24 @@ export class OpenapiMinuteData { ) { const query = this.getUpdateStockQuery(stockId, time); console.log(query); - const response = await getOpenApi( - this.url, - config, - query, - TR_IDS.MINUTE_DATA, - ); - let output; - if (response.output2) output = response.output2; - if (output && output[0] && isMinuteData(output[0])) { - this.saveMinuteData(stockId, output[0]); + try { + const response = await getOpenApi( + this.url, + config, + query, + TR_IDS.MINUTE_DATA, + ); + let output; + if (response.output2) output = response.output2; + if (output && output[0] && isMinuteData(output[0])) { + this.saveMinuteData(stockId, output); + } + } catch (error) { + console.error(error); } } + @UseFilters(OpenapiExceptionFilter) private async getMinuteDataChunk( chunk: Stock[], config: typeof openApiConfig, @@ -91,14 +112,18 @@ export class OpenapiMinuteData { } } - @Cron('* 9-14 * * 1-5') - @Cron('0-30 15 * * 1-5') + @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) + @UseFilters(OpenapiExceptionFilter) private getMinuteData() { + console.error('hello'); + if (process.env.NODE_ENV !== 'production') return; + console.error('not hello'); const configCount = openApiToken.configs.length; - const chunkSize = Math.ceil(this.stock.length / configCount); - + const stock = this.stock[this.flip % STOCK_CUT]; + this.flip++; + const chunkSize = Math.ceil(stock.length / configCount); for (let i = 0; i < configCount; i++) { - const chunk = this.stock.slice(i * chunkSize, (i + 1) * chunkSize); + const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); this.getMinuteDataChunk(chunk, openApiToken.configs[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index fa4b2b9b..5e35f3b9 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -45,12 +45,13 @@ export class OpenapiPeriodData { private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; public constructor(private readonly datasource: DataSource) { - this.getItemChartPriceCheck(); + //this.getItemChartPriceCheck(); } @Cron('0 1 * * 1-5') @UseFilters(OpenapiExceptionFilter) public async getItemChartPriceCheck() { + if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); const configCount = openApiToken.configs.length; diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 36936c89..e5c230cf 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,8 +1,7 @@ -import { Inject, UseFilters } from '@nestjs/common'; +import { Inject } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; import { logger } from '@/configs/logger.config'; @@ -59,7 +58,6 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') - @UseFilters(OpenapiExceptionFilter) private async initAccessToken() { const updatedConfig = await Promise.all( this.config.map(async (val) => { @@ -71,7 +69,6 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') - @UseFilters(OpenapiExceptionFilter) private async initWebSocketKey() { this.config.forEach(async (val) => { val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 9024497e..7d2f2d39 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,5 +1,8 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiMinuteData } from './api/openapiMinuteData.api'; +import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiScraperService } from './openapi-scraper.service'; import { Stock } from '@/stock/domain/stock.entity'; import { @@ -26,6 +29,11 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; ]), ], controllers: [], - providers: [OpenapiScraperService], + providers: [ + OpenapiPeriodData, + OpenapiMinuteData, + OpenapiDetailData, + OpenapiScraperService, + ], }) export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts index eb2b727b..7f1e2d81 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts @@ -3,17 +3,15 @@ import { DataSource } from 'typeorm'; import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; +import { openApiToken } from './api/openapiToken.api'; @Injectable() export class OpenapiScraperService { - private readonly openapiPeriodData: OpenapiPeriodData; - private readonly openapiMinuteData: OpenapiMinuteData; - private readonly openapiDetailData: OpenapiDetailData; - public constructor(private datasource: DataSource) { - if (process.env.NODE_ENV === 'production') { - this.openapiPeriodData = new OpenapiPeriodData(datasource); - this.openapiMinuteData = new OpenapiMinuteData(datasource); - this.openapiDetailData = new OpenapiDetailData(datasource); - } - } + private readonly token = openApiToken; + public constructor( + private datasource: DataSource, + private readonly openapiPeriodData: OpenapiPeriodData, + private readonly openapiMinuteData: OpenapiMinuteData, + private readonly openapiDetailData: OpenapiDetailData, + ) {} } From 54651bf34b2329e09ace9f1d3d74ede4ad5b3fad Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 18:28:52 +0900 Subject: [PATCH 043/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20minute=20data=20?= =?UTF-8?q?=EC=8B=9C=EA=B0=84=20=EC=B2=B4=ED=81=AC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiMinuteData.api.ts | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index e055de6e..529dd75b 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -42,18 +42,21 @@ export class OpenapiMinuteData { this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); i++; } - console.log(stock.length); - console.log(this.stock.length); - console.log(this.stock[0].length); } - private convertResToMinuteData(stockId: string, item: MinuteData) { + private convertResToMinuteData( + stockId: string, + item: MinuteData, + time: string, + ) { const stockPeriod = new StockData(); stockPeriod.stock = { id: stockId } as Stock; stockPeriod.startTime = new Date( parseInt(item.stck_bsop_date.slice(0, 4)), parseInt(item.stck_bsop_date.slice(4, 6)) - 1, parseInt(item.stck_bsop_date.slice(6, 8)), + parseInt(time.slice(0, 2)), + parseInt(time.slice(2, 4)), ); stockPeriod.close = parseInt(item.stck_prpr); stockPeriod.open = parseInt(item.stck_oprc); @@ -64,10 +67,20 @@ export class OpenapiMinuteData { return stockPeriod; } - private async saveMinuteData(stockId: string, item: MinuteData[]) { + private isMarketOpenTime(time: string) { + const numberTime = parseInt(time); + return numberTime >= 90000 && numberTime <= 153000; + } + + private async saveMinuteData( + stockId: string, + item: MinuteData[], + time: string, + ) { const manager = this.datasource.manager; + if (this.isMarketOpenTime(time)) return; const stockPeriod = item.map((val) => - this.convertResToMinuteData(stockId, val), + this.convertResToMinuteData(stockId, val, time), ); manager.save(this.entity, stockPeriod); } @@ -78,7 +91,6 @@ export class OpenapiMinuteData { config: typeof openApiConfig, ) { const query = this.getUpdateStockQuery(stockId, time); - console.log(query); try { const response = await getOpenApi( this.url, @@ -89,7 +101,7 @@ export class OpenapiMinuteData { let output; if (response.output2) output = response.output2; if (output && output[0] && isMinuteData(output[0])) { - this.saveMinuteData(stockId, output); + this.saveMinuteData(stockId, output, time); } } catch (error) { console.error(error); @@ -115,9 +127,7 @@ export class OpenapiMinuteData { @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) @UseFilters(OpenapiExceptionFilter) private getMinuteData() { - console.error('hello'); if (process.env.NODE_ENV !== 'production') return; - console.error('not hello'); const configCount = openApiToken.configs.length; const stock = this.stock[this.flip % STOCK_CUT]; this.flip++; From 433cc93160f1678ec1bfe9536105e99b740d47c4 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 21:15:00 +0900 Subject: [PATCH 044/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=9B=B9=EC=86=8C?= =?UTF-8?q?=EC=BC=93=20=EA=B2=8C=EC=9D=B4=ED=8A=B8=20=EA=B2=BD=EB=A1=9C=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.gateway.ts | 2 +- packages/backend/src/stock/stock.gateway.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/chat/chat.gateway.ts b/packages/backend/src/chat/chat.gateway.ts index 65474edb..0fd6a1f0 100644 --- a/packages/backend/src/chat/chat.gateway.ts +++ b/packages/backend/src/chat/chat.gateway.ts @@ -35,7 +35,7 @@ interface chatResponse { createdAt: Date; } -@WebSocketGateway({ namespace: 'chat' }) +@WebSocketGateway({ namespace: '/api/chat/realtime' }) @UseFilters(WebSocketExceptionFilter) export class ChatGateway implements OnGatewayConnection { @WebSocketServer() diff --git a/packages/backend/src/stock/stock.gateway.ts b/packages/backend/src/stock/stock.gateway.ts index cf98907c..1f4ab32d 100644 --- a/packages/backend/src/stock/stock.gateway.ts +++ b/packages/backend/src/stock/stock.gateway.ts @@ -1,14 +1,14 @@ import { + ConnectedSocket, + MessageBody, + SubscribeMessage, WebSocketGateway, WebSocketServer, - SubscribeMessage, - MessageBody, - ConnectedSocket, } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; @WebSocketGateway({ - namespace: '/stock/realtime', + namespace: '/api/stock/realtime', }) export class StockGateway { @WebSocketServer() From 3b543a6d7d8aac75f33c727b78f314dad67bdc9b Mon Sep 17 00:00:00 2001 From: sunghwki Date: Wed, 20 Nov 2024 22:33:11 +0900 Subject: [PATCH 045/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20output=20?= =?UTF-8?q?=ED=8C=8C=EC=9D=BC=20=EC=82=AD=EC=A0=9C,=20DI,=20=EB=A1=9C?= =?UTF-8?q?=EC=A7=81=20=EB=B3=80=EA=B2=BD(isMarketOpenTime=EC=9D=B4=20?= =?UTF-8?q?=EC=A0=95=EB=B0=98=EB=8C=80=EB=A1=9C=20=EA=B5=AC=ED=98=84?= =?UTF-8?q?=EB=90=98=EC=96=B4=20=EC=9E=88=EC=97=88=EC=9D=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- output | 99 ------------------- .../backend/src/chat/like.service.spec.ts | 8 +- .../Decorator/openapiException.filter.ts | 3 +- .../openapi/api/openapiDetailData.api.ts | 2 +- .../openapi/api/openapiMinuteData.api.ts | 2 +- .../openapi/type/openapiDetailData.type.ts | 1 + 6 files changed, 9 insertions(+), 106 deletions(-) delete mode 100644 output diff --git a/output b/output deleted file mode 100644 index cdd2d49a..00000000 --- a/output +++ /dev/null @@ -1,99 +0,0 @@ -yarn workspace v1.22.22 -yarn run v1.22.22 -$ nest start --watch -[5:32:54 PM] Starting compilation in watch mode... - -[5:32:56 PM] Found 0 errors. Watching for file changes. - -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [NestFactory] Starting Nest application... -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] AppModule dependencies initialized +41ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ScraperModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] SessionModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ConfigHostModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] WinstonModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] DiscoveryModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ConfigModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ScheduleModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmCoreModule dependencies initialized +57ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] KoreaStockInfoModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] OpenapiScraperModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] UserModule dependencies initialized +1ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] StockModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] ChatModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [InstanceLoader] AuthModule dependencies initialized +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [WebSocketsController] StockGateway subscribed to the "connectStock" message +13ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [WebSocketsController] ChatGateway subscribed to the "chat" message +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] StockController {/api/stock}: +1ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/view, POST} route +1ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user, POST} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user, DELETE} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/user/ownership, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/minutely, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/daily, GET} route +1ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/weekly, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/mothly, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/yearly, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/detail, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topViews, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topGainers, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/stock/topLosers, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] UserController {/api/user}: +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, PATCH} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RoutesResolver] GoogleAuthController {/api/auth/google}: +1ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/login, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/redirect, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [RouterExplorer] Mapped {/api/auth/google/status, GET} route +0ms -[Nest] 26105 - 11/20/2024, 5:32:57 PM  LOG [NestApplication] Nest application successfully started +5ms -[5:34:34 PM] File change detected. Starting incremental compilation... - -[5:34:34 PM] Found 0 errors. Watching for file changes. - -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [NestFactory] Starting Nest application... -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] AppModule dependencies initialized +40ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ScraperModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] SessionModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ConfigHostModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] WinstonModule dependencies initialized +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] DiscoveryModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ConfigModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ScheduleModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmCoreModule dependencies initialized +56ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] KoreaStockInfoModule dependencies initialized +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] TypeOrmModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] OpenapiScraperModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] UserModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] StockModule dependencies initialized +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] ChatModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [InstanceLoader] AuthModule dependencies initialized +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [WebSocketsController] StockGateway subscribed to the "connectStock" message +11ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [WebSocketsController] ChatGateway subscribed to the "chat" message +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] StockController {/api/stock}: +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/view, POST} route +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user, POST} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user, DELETE} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/user/ownership, GET} route +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/minutely, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/daily, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/weekly, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/mothly, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/yearly, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/:stockId/detail, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topViews, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topGainers, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/stock/topLosers, GET} route +1ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] UserController {/api/user}: +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, PATCH} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/user/:id/theme, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RoutesResolver] GoogleAuthController {/api/auth/google}: +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/login, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/redirect, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [RouterExplorer] Mapped {/api/auth/google/status, GET} route +0ms -[Nest] 26726 - 11/20/2024, 5:34:35 PM  LOG [NestApplication] Nest application successfully started +5ms diff --git a/packages/backend/src/chat/like.service.spec.ts b/packages/backend/src/chat/like.service.spec.ts index 1099b7bd..5df6fed8 100644 --- a/packages/backend/src/chat/like.service.spec.ts +++ b/packages/backend/src/chat/like.service.spec.ts @@ -1,10 +1,10 @@ -import { createDataSourceMock } from '@/user/user.service.spec'; import { DataSource } from 'typeorm'; -import { LikeService } from '@/chat/like.service'; import { Chat } from '@/chat/domain/chat.entity'; +import { Like } from '@/chat/domain/like.entity'; +import { LikeService } from '@/chat/like.service'; import { Stock } from '@/stock/domain/stock.entity'; import { User } from '@/user/domain/user.entity'; -import { Like } from '@/chat/domain/like.entity'; +import { createDataSourceMock } from '@/user/user.service.spec'; function createChat(): Chat { return { @@ -65,4 +65,4 @@ describe('LikeService 테스트', () => { expect(response.likeCount).toBe(0); }); -}); \ No newline at end of file +}); diff --git a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts index e86915b3..a6c45ae9 100644 --- a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts +++ b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts @@ -3,13 +3,14 @@ import { Catch, HttpException, HttpStatus, + Inject, } from '@nestjs/common'; import { Logger } from 'winston'; import { OpenapiException } from '../util/openapiCustom.error'; @Catch() export class OpenapiExceptionFilter implements ExceptionFilter { - private readonly logger = new Logger(); + constructor(@Inject('winston') private readonly logger: Logger) {} catch(exception: unknown) { const status = diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 1745d0d6..2f7678ff 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -139,7 +139,7 @@ export class OpenapiDetailData { private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { let delay = 0; - for (const stock of chunk) { + for await (const stock of chunk) { setTimeout(() => this.getDetailDataDelay(stock, conf), delay); delay += this.intervals; } diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index 529dd75b..e050e3cd 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -78,7 +78,7 @@ export class OpenapiMinuteData { time: string, ) { const manager = this.datasource.manager; - if (this.isMarketOpenTime(time)) return; + if (!this.isMarketOpenTime(time)) return; const stockPeriod = item.map((val) => this.convertResToMinuteData(stockId, val, time), ); diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts index f05edcea..772d8952 100644 --- a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -6,6 +6,7 @@ export type DetailDataQuery = { fid_input_iscd: string; fid_div_cls_code: '0' | '1'; }; + export type FinancialData = { stac_yymm: string; // 결산 년월 grs: string; // 매출액 증가율 From d6e573948c98a8be3b2bbc6c2abafdcb1bcb5937 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Wed, 20 Nov 2024 23:03:18 +0900 Subject: [PATCH 046/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20=EC=9B=B9=20?= =?UTF-8?q?=EC=86=8C=EC=BC=93=20=EB=B9=84=EC=96=B4=EC=9E=88=EB=8A=94=20?= =?UTF-8?q?=EC=BF=A0=ED=82=A4=20=EC=97=90=EB=9F=AC=20=EB=AC=B8=EC=A0=9C=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/auth/session/websocketSession.service.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/auth/session/websocketSession.service.ts b/packages/backend/src/auth/session/websocketSession.service.ts index 10af73c2..c6c248cc 100644 --- a/packages/backend/src/auth/session/websocketSession.service.ts +++ b/packages/backend/src/auth/session/websocketSession.service.ts @@ -7,16 +7,21 @@ export class WebsocketSessionService { constructor(private readonly sessionStore: MemoryStore) {} async getAuthenticatedUser(socket: Socket) { - const cookieValue = websocketCookieParse(socket); - const session = await this.getSession(cookieValue); - return session ? session.passport.user : undefined; + try { + const cookieValue = websocketCookieParse(socket); + const session = await this.getSession(cookieValue); + return session ? session.passport.user : null; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return null; + } } private getSession(cookieValue: string) { - return new Promise((resolve) => { + return new Promise((resolve) => { this.sessionStore.get(cookieValue, (err: Error, session) => { if (err || !session) { - resolve(undefined); + resolve(null); } resolve(session as PassportSession); }); From 4dcaa3109ebbf50869e7c8f999de641cf29f89e5 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 01:02:23 +0900 Subject: [PATCH 047/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20openap?= =?UTF-8?q?i=20scraper=20service=20=EC=97=90=EC=84=9C=20=EC=95=88=20?= =?UTF-8?q?=EC=93=B0=EC=9D=B4=EB=8A=94=20=ED=81=B4=EB=9E=98=EC=8A=A4=20?= =?UTF-8?q?=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts | 0 packages/backend/src/scraper/openapi/openapi-scraper.service.ts | 2 -- 2 files changed, 2 deletions(-) create mode 100644 packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts index 7f1e2d81..52c90179 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts @@ -3,11 +3,9 @@ import { DataSource } from 'typeorm'; import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; -import { openApiToken } from './api/openapiToken.api'; @Injectable() export class OpenapiScraperService { - private readonly token = openApiToken; public constructor( private datasource: DataSource, private readonly openapiPeriodData: OpenapiPeriodData, From 31b77ffb81c95434a73e72c1eaa637c4926f623e Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 13:02:38 +0900 Subject: [PATCH 048/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20injectable?= =?UTF-8?q?=ED=95=98=EA=B2=8C=20=EB=B3=80=EA=B2=BD,=20=EC=9D=B4=EC=A0=84?= =?UTF-8?q?=20websocket=20=EC=BD=94=EB=93=9C=20=EC=82=AD=EC=A0=9C=E3=85=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/openapi/api/openapiDetailData.api.ts | 3 ++- .../src/scraper/openapi/api/openapiLiveData.api.ts | 0 .../src/scraper/openapi/api/openapiToken.api.ts | 11 +++++++---- 3 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 2f7678ff..7b365780 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -1,4 +1,4 @@ -import { UseFilters } from '@nestjs/common'; +import { Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Between, DataSource } from 'typeorm'; import { openApiConfig } from '../config/openapi.config'; @@ -18,6 +18,7 @@ import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; import { StockDetail } from '@/stock/domain/stockDetail.entity'; +@Injectable() export class OpenapiDetailData { private readonly financialUrl: string = '/uapi/domestic-stock/v1/finance/financial-ratio'; diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index e5c230cf..13ac793c 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,7 +1,8 @@ -import { Inject } from '@nestjs/common'; +import { Inject, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; import { logger } from '@/configs/logger.config'; @@ -34,6 +35,7 @@ class OpenapiTokenApi { return this.config; } + @UseFilters(OpenapiExceptionFilter) private async initAuthenValue() { const delay = 60000; const delayMinute = delay / 1000 / 60; @@ -50,9 +52,10 @@ class OpenapiTokenApi { this.logger.warn( `Request failed. Retrying in ${delayMinute} minute...`, ); - await new Promise((resolve) => setTimeout(resolve, delay)); - await this.initAccessToken(); - await this.initWebSocketKey(); + setTimeout(async () => { + await this.initAccessToken(); + await this.initWebSocketKey(); + }, delay); } } } From 3978d00bc16fbbe6354e7790c77024253be3a8ef Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 13:24:56 +0900 Subject: [PATCH 049/112] =?UTF-8?q?=E2=9C=A8=20feat:=20websocket=20?= =?UTF-8?q?=EC=97=B0=EA=B2=B0=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/websocketClient.service.ts | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 packages/backend/src/scraper/openapi/websocketClient.service.ts diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts new file mode 100644 index 00000000..5071bff3 --- /dev/null +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -0,0 +1,50 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { WebSocket } from 'ws'; + +@Injectable() +export class WebsocketClient { + private client: WebSocket; + private readonly reconnectInterval = 60000; + private readonly url = + process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; + + constructor(@Inject('winston') private readonly logger: Logger) { + this.connect(); + } + + @Cron('0 2 * * 1-5') + private connect() { + this.client = new WebSocket(this.url); + + this.client.on('open', () => { + this.logger.log('WebSocket connection established'); + this.sendMessage('Initial message'); + }); + + this.client.on('message', (data: any) => { + this.logger.log(`Received message: ${data}`); + }); + + this.client.on('close', () => { + this.logger.warn( + `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, + ); + setTimeout(() => this.connect(), this.reconnectInterval); + }); + + this.client.on('error', (error: any) => { + this.logger.error(`WebSocket error: ${error.message}`); + }); + } + + private sendMessage(message: string) { + if (this.client.readyState === WebSocket.OPEN) { + this.client.send(message); + this.logger.log(`Sent message: ${message}`); + } else { + this.logger.warn('WebSocket is not open. Message not sent.'); + } + } +} From d157f1ef2c536eb0ea27f4c0d4b8acdc94166caf Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 15:35:13 +0900 Subject: [PATCH 050/112] =?UTF-8?q?=E2=9C=A8=20feat:=20live=20data=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + packages/backend/.gitignore | 4 + packages/backend/package.json | 4 +- .../openapi/api/openapiLiveData.api.ts | 107 ++++++++++++ .../scraper/openapi/api/openapiToken.api.ts | 10 +- .../scraper/openapi/openapi-scraper.module.ts | 4 + .../openapi/type/openapiLiveData.type.ts | 152 ++++++++++++++++++ .../scraper/openapi/util/openapiUtil.api.ts | 16 ++ .../openapi/websocketClient.service.ts | 37 ++++- yarn.lock | 25 +-- 10 files changed, 340 insertions(+), 20 deletions(-) create mode 100644 packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts diff --git a/.gitignore b/.gitignore index 66b03b45..57004910 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # vscode setting .vscode + diff --git a/packages/backend/.gitignore b/packages/backend/.gitignore index 2f6b899c..cd939265 100644 --- a/packages/backend/.gitignore +++ b/packages/backend/.gitignore @@ -54,3 +54,7 @@ pids # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# backup file +.backup +.bak diff --git a/packages/backend/package.json b/packages/backend/package.json index c41b6046..26da1fd9 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,11 +25,11 @@ "@nestjs/core": "^10.0.0", "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.0.0", - "@nestjs/platform-socket.io": "^10.4.7", + "@nestjs/platform-socket.io": "^10.4.8", "@nestjs/schedule": "^4.1.1", "@nestjs/swagger": "^8.0.5", "@nestjs/typeorm": "^10.0.2", - "@nestjs/websockets": "^10.4.7", + "@nestjs/websockets": "^10.4.8", "axios": "^1.7.7", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts new file mode 100644 index 00000000..f0589c93 --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -0,0 +1,107 @@ +import { Inject } from '@nestjs/common'; +import { EntityManager } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { + MessageResponse, + StockData, + isMessageResponse, + parseStockData, +} from '../type/openapiLiveData.type'; +import { decryptAES256 } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { KospiStock } from '@/stock/domain/kospiStock.entity'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; + +export class OpenapiLiveData { + public readonly TR_ID: string = 'H0STCNT0'; + private readonly WEBSOCKET_MAX: number = 40; + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly manager: EntityManager, + ) {} + + public async getMessage(): Promise { + const kospi = await this.getKospiStockId(); + const config = openApiToken.configs; + const configLength = config.length; + const ret: string[] = []; + + for (let i = 0; i < configLength; i++) { + const stocks = kospi.splice( + i * this.WEBSOCKET_MAX, + (i + 1) * this.WEBSOCKET_MAX, + ); + for (const stock of stocks) { + ret.push(this.convertObjectToMessage(config[i], stock.id!)); + } + } + + return ret; + } + + private convertObjectToMessage( + config: typeof openApiConfig, + stockId: string, + ): string { + const message = { + header: { + approval_key: config.STOCK_WEBSOCKET_KEY!, + custtype: 'P', + tr_type: '1', + 'content-type': 'utf-8', + }, + body: { + input: { + tr_id: this.TR_ID, + tr_key: stockId, + }, + }, + }; + return JSON.stringify(message); + } + + private async getKospiStockId() { + const kospi = await this.manager.find(KospiStock); + return kospi; + } + + private async saveLiveData(data: StockLiveData) { + await this.manager.save(StockLiveData, data); + } + + private convertLiveData(message: string[]): StockLiveData { + const stockData: StockData = parseStockData(message); + const stockLiveData = new StockLiveData(); + stockLiveData.currentPrice = parseFloat(stockData.STCK_PRPR); + stockLiveData.changeRate = parseFloat(stockData.PRDY_CTRT); + stockLiveData.volume = parseInt(stockData.CNTG_VOL); + stockLiveData.high = parseFloat(stockData.STCK_HGPR); + stockLiveData.low = parseFloat(stockData.STCK_LWPR); + stockLiveData.open = parseFloat(stockData.STCK_OPRC); + stockLiveData.previousClose = parseFloat(stockData.WGHN_AVRG_STCK_PRC); + stockLiveData.updatedAt = new Date(); + + return stockLiveData; + } + + public async output(message: Buffer, iv?: string, key?: string) { + const str = message.toString(); + if (str.split('|').length < 3) return; + const parsed = str.split('|'); + if (parsed.length > 0) { + if (parsed[0] == '1' && iv && key) + parsed[4] = decryptAES256(parsed[4], iv, key); + if (parsed[1] !== this.TR_ID) return; + const stockData = parsed[4].split('^'); + const length = stockData.length / parseInt(parsed[3]); + const size = parseInt(parsed[2]); + const i = 0; + while (i < size) { + const data = stockData.splice(i * length, (i + 1) * length); + const liveData = this.convertLiveData(data); + this.saveLiveData(liveData); + } + } + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 13ac793c..275730bc 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -73,9 +73,13 @@ class OpenapiTokenApi { @Cron('50 0 * * 1-5') private async initWebSocketKey() { - this.config.forEach(async (val) => { - val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; - }); + const updatedConfig = await Promise.all( + this.config.map(async (val) => { + val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; + return val; + }), + ); + this.config = updatedConfig; } private async getToken(config: typeof openApiConfig): Promise { diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 7d2f2d39..cb45c91c 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,9 +1,11 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiScraperService } from './openapi-scraper.service'; +import { WebsocketClient } from './websocketClient.service'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily, @@ -34,6 +36,8 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; OpenapiMinuteData, OpenapiDetailData, OpenapiScraperService, + OpenapiLiveData, + WebsocketClient, ], }) export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts new file mode 100644 index 00000000..d8041e7b --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts @@ -0,0 +1,152 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable max-lines-per-function */ + +export type StockData = { + MKSC_SHRN_ISCD: string; // 유가증권 단축 종목코드 + STCK_CNTG_HOUR: string; // 주식 체결 시간 + STCK_PRPR: string; // 주식 현재가 + PRDY_VRSS_SIGN: string; // 전일 대비 부호 + PRDY_VRSS: string; // 전일 대비 + PRDY_CTRT: string; // 전일 대비율 + WGHN_AVRG_STCK_PRC: string; // 가중 평균 주식 가격 + STCK_OPRC: string; // 주식 시가 + STCK_HGPR: string; // 주식 최고가 + STCK_LWPR: string; // 주식 최저가 + ASKP1: string; // 매도호가1 + BIDP1: string; // 매수호가1 + CNTG_VOL: string; // 체결 거래량 + ACML_VOL: string; // 누적 거래량 + ACML_TR_PBMN: string; // 누적 거래 대금 + SELN_CNTG_CSNU: string; // 매도 체결 건수 + SHNU_CNTG_CSNU: string; // 매수 체결 건수 + NTBY_CNTG_CSNU: string; // 순매수 체결 건수 + CTTR: string; // 체결강도 + SELN_CNTG_SMTN: string; // 총 매도 수량 + SHNU_CNTG_SMTN: string; // 총 매수 수량 + CCLD_DVSN: string; // 체결구분 + SHNU_RATE: string; // 매수비율 + PRDY_VOL_VRSS_ACML_VOL_RATE: string; // 전일 거래량 대비 등락율 + OPRC_HOUR: string; // 시가 시간 + OPRC_VRSS_PRPR_SIGN: string; // 시가대비구분 + OPRC_VRSS_PRPR: string; // 시가대비 + HGPR_HOUR: string; // 최고가 시간 + HGPR_VRSS_PRPR_SIGN: string; // 고가대비구분 + HGPR_VRSS_PRPR: string; // 고가대비 + LWPR_HOUR: string; // 최저가 시간 + LWPR_VRSS_PRPR_SIGN: string; // 저가대비구분 + LWPR_VRSS_PRPR: string; // 저가대비 + BSOP_DATE: string; // 영업 일자 + NEW_MKOP_CLS_CODE: string; // 신 장운영 구분 코드 + TRHT_YN: string; // 거래정지 여부 + ASKP_RSQN1: string; // 매도호가 잔량1 + BIDP_RSQN1: string; // 매수호가 잔량1 + TOTAL_ASKP_RSQN: string; // 총 매도호가 잔량 + TOTAL_BIDP_RSQN: string; // 총 매수호가 잔량 + VOL_TNRT: string; // 거래량 회전율 + PRDY_SMNS_HOUR_ACML_VOL: string; // 전일 동시간 누적 거래량 + PRDY_SMNS_HOUR_ACML_VOL_RATE: string; // 전일 동시간 누적 거래량 비율 + HOUR_CLS_CODE: string; // 시간 구분 코드 + MRKT_TRTM_CLS_CODE: string; // 임의종료구분코드 + VI_STND_PRC: string; // 정적VI발동기준가 +}; + +export function parseStockData(message: string[]): StockData { + return { + MKSC_SHRN_ISCD: message[0], + STCK_CNTG_HOUR: message[1], + STCK_PRPR: message[2], + PRDY_VRSS_SIGN: message[3], + PRDY_VRSS: message[4], + PRDY_CTRT: message[5], + WGHN_AVRG_STCK_PRC: message[6], + STCK_OPRC: message[7], + STCK_HGPR: message[8], + STCK_LWPR: message[9], + ASKP1: message[10], + BIDP1: message[11], + CNTG_VOL: message[12], + ACML_VOL: message[13], + ACML_TR_PBMN: message[14], + SELN_CNTG_CSNU: message[15], + SHNU_CNTG_CSNU: message[16], + NTBY_CNTG_CSNU: message[17], + CTTR: message[18], + SELN_CNTG_SMTN: message[19], + SHNU_CNTG_SMTN: message[20], + CCLD_DVSN: message[21], + SHNU_RATE: message[22], + PRDY_VOL_VRSS_ACML_VOL_RATE: message[23], + OPRC_HOUR: message[24], + OPRC_VRSS_PRPR_SIGN: message[25], + OPRC_VRSS_PRPR: message[26], + HGPR_HOUR: message[27], + HGPR_VRSS_PRPR_SIGN: message[28], + HGPR_VRSS_PRPR: message[29], + LWPR_HOUR: message[30], + LWPR_VRSS_PRPR_SIGN: message[31], + LWPR_VRSS_PRPR: message[32], + BSOP_DATE: message[33], + NEW_MKOP_CLS_CODE: message[34], + TRHT_YN: message[35], + ASKP_RSQN1: message[36], + BIDP_RSQN1: message[37], + TOTAL_ASKP_RSQN: message[38], + TOTAL_BIDP_RSQN: message[39], + VOL_TNRT: message[40], + PRDY_SMNS_HOUR_ACML_VOL: message[41], + PRDY_SMNS_HOUR_ACML_VOL_RATE: message[42], + HOUR_CLS_CODE: message[43], + MRKT_TRTM_CLS_CODE: message[44], + VI_STND_PRC: message[45], + }; +} + +export type OpenApiMessage = { + header: { + approval_key: string; + custtype: string; + tr_type: string; + 'content-type': string; + }; + body: { + input: { + tr_id: string; + tr_key: string; + }; + }; +}; + +export type MessageResponse = { + header: { + tr_id: string; + tr_key: string; + encrypt: string; + }; + body: { + rt_cd: string; + msg_cd: string; + msg1: string; + output?: { + iv: string; + key: string; + }; + }; +}; + +export function isMessageResponse(data: any): data is MessageResponse { + return ( + typeof data === 'object' && + data !== null && + typeof data.header === 'object' && + data.header !== null && + typeof data.header.tr_id === 'object' && + typeof data.header.tr_key === 'object' && + typeof data.header.encrypt === 'object' && + typeof data.body === 'object' && + data.body !== null && + typeof data.body.rt_cd === 'object' && + typeof data.body.msg_cd === 'object' && + typeof data.body.msg1 === 'object' && + typeof data.body.output === 'object' + ); +} diff --git a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts index 7c177443..4ec0b29d 100644 --- a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts +++ b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any*/ +import * as crypto from 'crypto'; import { HttpStatus } from '@nestjs/common'; import axios from 'axios'; import { openApiConfig } from '../config/openapi.config'; @@ -77,6 +78,20 @@ const getCurrentTime = () => { const seconds = String(now.getSeconds()).padStart(2, '0'); return `${hours}${minutes}${seconds}`; }; +const decryptAES256 = ( + encryptedText: string, + key: string, + iv: string, +): string => { + const decipher = crypto.createDecipheriv( + 'aes-256-cbc', + Buffer.from(key, 'hex'), + Buffer.from(iv, 'hex'), + ); + let decrypted = decipher.update(encryptedText, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + return decrypted; +}; export { postOpenApi, @@ -84,4 +99,5 @@ export { getTodayDate, getPreviousDate, getCurrentTime, + decryptAES256, }; diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 5071bff3..c91b0e71 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -1,7 +1,9 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { Inject, Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; +import { Logger } from 'winston'; import { WebSocket } from 'ws'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; @Injectable() export class WebsocketClient { @@ -10,21 +12,36 @@ export class WebsocketClient { private readonly url = process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; - constructor(@Inject('winston') private readonly logger: Logger) { + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly openapiLiveData: OpenapiLiveData, + ) { this.connect(); } + // TODO : subscribe 구조로 리팩토링 + private subscribe() {} + @Cron('0 2 * * 1-5') private connect() { this.client = new WebSocket(this.url); this.client.on('open', () => { - this.logger.log('WebSocket connection established'); - this.sendMessage('Initial message'); + this.logger.info('WebSocket connection established'); + this.openapiLiveData.getMessage().then((val) => { + val.forEach((message) => this.sendMessage(message)); + }); }); this.client.on('message', (data: any) => { - this.logger.log(`Received message: ${data}`); + this.logger.info(`Received message: ${data}`); + const message = JSON.parse(data); + if (message.header && message.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(message)}`); + this.sendPong(); + return; + } + this.openapiLiveData.output(data); }); this.client.on('close', () => { @@ -39,10 +56,18 @@ export class WebsocketClient { }); } + private sendPong() { + const pongMessage = { + header: { tr_id: 'PINGPONG', datetime: new Date().toISOString() }, + }; + this.client.send(JSON.stringify(pongMessage)); + this.logger.info(`Sent PONG: ${JSON.stringify(pongMessage)}`); + } + private sendMessage(message: string) { if (this.client.readyState === WebSocket.OPEN) { this.client.send(message); - this.logger.log(`Sent message: ${message}`); + this.logger.info(`Sent message: ${message}`); } else { this.logger.warn('WebSocket is not open. Message not sent.'); } diff --git a/yarn.lock b/yarn.lock index b8b27206..624dd31a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1287,10 +1287,10 @@ multer "1.4.4-lts.1" tslib "2.7.0" -"@nestjs/platform-socket.io@^10.4.7": - version "10.4.7" - resolved "https://registry.yarnpkg.com/@nestjs/platform-socket.io/-/platform-socket.io-10.4.7.tgz#0c22c204e72aba83dff5b517dcd4802fc0f17594" - integrity sha512-CpmrqswpD/O4SyF/IUzKj14BUf0eTLyDja9svPCRIJX8AdF47mKCMbz5vtU6vpJtxVnq1e1Xd+xcdZ6FIf6HtQ== +"@nestjs/platform-socket.io@^10.4.8": + version "10.4.8" + resolved "https://registry.yarnpkg.com/@nestjs/platform-socket.io/-/platform-socket.io-10.4.8.tgz#cf483794f3b1831d804a3ac3a3f7b999664489d4" + integrity sha512-KzCL+P037HiaW3iODueJ/vw5a8bSr6uIturgGSuRz6c8WR+SfqKC6jNXw0JTW5NVqEqX8tOunEVXoI3MFnWz/w== dependencies: socket.io "4.8.0" tslib "2.7.0" @@ -1340,10 +1340,10 @@ dependencies: uuid "9.0.1" -"@nestjs/websockets@^10.4.7": - version "10.4.7" - resolved "https://registry.yarnpkg.com/@nestjs/websockets/-/websockets-10.4.7.tgz#20d4da5e38a1f1dff866f780e694c907eaa23b8f" - integrity sha512-ajuoptYLYm+l3+KtaA9Ed+cO9yB34PtBE8UObavRT8Euh/f7QfeJiKcrU3+BQSAiTWM3nF2qfuV4CfEkP9uKuw== +"@nestjs/websockets@^10.4.8": + version "10.4.8" + resolved "https://registry.yarnpkg.com/@nestjs/websockets/-/websockets-10.4.8.tgz#9c2b982059e850a56999f56c87ac3a88acbce4ea" + integrity sha512-IpObWsZvjjUxmBuIF/AkcyXrFFzwNYNsw2reZXHy7C31wJsYAjwr6rHMSRGyqsxfqTA2DqjCczorewM6BAEXig== dependencies: iterare "1.2.1" object-hash "3.0.0" @@ -2269,6 +2269,13 @@ resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.12.2.tgz#760329e756e18a4aab82fc502b51ebdfebbe49f5" integrity sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA== +"@types/ws@^8.5.13": + version "8.5.13" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.13.tgz#6414c280875e2691d0d1e080b05addbf5cb91e20" + integrity sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -8971,7 +8978,7 @@ write-file-atomic@^4.0.2: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@^8.2.3: +ws@^8.18.0, ws@^8.2.3: version "8.18.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== From f00641d700220d46c03c2dfa59c2c526970de686 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 15:38:10 +0900 Subject: [PATCH 051/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20=EA=B8=B0?= =?UTF-8?q?=EC=A1=B4=20=EC=BD=94=EB=93=9C=EC=97=90=EC=84=9C=20=20object=20?= =?UTF-8?q?=EA=B2=80=EC=82=AC=20=EB=A1=9C=EC=A7=81=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/scraper/openapi/api/openapiLiveData.api.ts | 4 +--- .../backend/src/scraper/openapi/websocketClient.service.ts | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index f0589c93..5e4e9acf 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -86,9 +86,7 @@ export class OpenapiLiveData { } public async output(message: Buffer, iv?: string, key?: string) { - const str = message.toString(); - if (str.split('|').length < 3) return; - const parsed = str.split('|'); + const parsed = message.toString().split('|'); if (parsed.length > 0) { if (parsed[0] == '1' && iv && key) parsed[4] = decryptAES256(parsed[4], iv, key); diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index c91b0e71..d2b4a9d8 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -41,6 +41,9 @@ export class WebsocketClient { this.sendPong(); return; } + if (message.header && message.header.tr_id === 'H0STCNT0') { + return; + } this.openapiLiveData.output(data); }); From 51cf7202cc03eb34265761de5df03f502279e575 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 15:39:57 +0900 Subject: [PATCH 052/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20eslint?= =?UTF-8?q?=20=EC=A4=80=EC=88=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiLiveData.api.ts | 7 +---- .../openapi/websocketClient.service.ts | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 5e4e9acf..77a31c67 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -2,12 +2,7 @@ import { Inject } from '@nestjs/common'; import { EntityManager } from 'typeorm'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; -import { - MessageResponse, - StockData, - isMessageResponse, - parseStockData, -} from '../type/openapiLiveData.type'; +import { StockData, parseStockData } from '../type/openapiLiveData.type'; import { decryptAES256 } from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; import { KospiStock } from '@/stock/domain/kospiStock.entity'; diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index d2b4a9d8..47bafad1 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -22,6 +22,20 @@ export class WebsocketClient { // TODO : subscribe 구조로 리팩토링 private subscribe() {} + private message(data: any) { + this.logger.info(`Received message: ${data}`); + const message = JSON.parse(data); + if (message.header && message.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(message)}`); + this.sendPong(); + return; + } + if (message.header && message.header.tr_id === 'H0STCNT0') { + return; + } + this.openapiLiveData.output(data); + } + @Cron('0 2 * * 1-5') private connect() { this.client = new WebSocket(this.url); @@ -34,17 +48,7 @@ export class WebsocketClient { }); this.client.on('message', (data: any) => { - this.logger.info(`Received message: ${data}`); - const message = JSON.parse(data); - if (message.header && message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(message)}`); - this.sendPong(); - return; - } - if (message.header && message.header.tr_id === 'H0STCNT0') { - return; - } - this.openapiLiveData.output(data); + this.message(data); }); this.client.on('close', () => { From 6135a8792d76b8c73fe39fe7f4a58c943c1fe462 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 21 Nov 2024 15:42:35 +0900 Subject: [PATCH 053/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=A3=BC=EC=8B=9D?= =?UTF-8?q?=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=A1=B0=ED=9A=8C=20=EC=97=94?= =?UTF-8?q?=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=ED=86=B5=ED=95=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../stock/decorator/stockData.decorator.ts | 14 ++- .../backend/src/stock/stock.controller.ts | 119 ++++++++++-------- 2 files changed, 78 insertions(+), 55 deletions(-) diff --git a/packages/backend/src/stock/decorator/stockData.decorator.ts b/packages/backend/src/stock/decorator/stockData.decorator.ts index 19eb3969..d48dccc1 100644 --- a/packages/backend/src/stock/decorator/stockData.decorator.ts +++ b/packages/backend/src/stock/decorator/stockData.decorator.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable max-lines-per-function */ -import { applyDecorators } from '@nestjs/common'; -import { ApiOperation, ApiParam, ApiQuery, ApiResponse } from '@nestjs/swagger'; -import { StockDataResponse } from '../dto/stockData.response'; +import { applyDecorators } from "@nestjs/common"; +import { ApiOperation, ApiParam, ApiQuery, ApiResponse } from "@nestjs/swagger"; +import { StockDataResponse } from "../dto/stockData.response"; export function ApiGetStockData(summary: string, type: string) { return applyDecorators( @@ -22,6 +22,14 @@ export function ApiGetStockData(summary: string, type: string) { type: String, format: 'date-time', }), + ApiQuery({ + name: 'timeunit', + required: false, + description: '시간 단위', + example: 'minute', + type: String, + enum: ['minute', 'day', 'week', 'month', 'year'], + }), ApiResponse({ status: 200, description: `주식의 ${type} 단위 데이터 성공적으로 조회`, diff --git a/packages/backend/src/stock/stock.controller.ts b/packages/backend/src/stock/stock.controller.ts index 5aaf6707..8d75c87c 100644 --- a/packages/backend/src/stock/stock.controller.ts +++ b/packages/backend/src/stock/stock.controller.ts @@ -32,6 +32,7 @@ import { StockDetailService } from './stockDetail.service'; import SessionGuard from '@/auth/session/session.guard'; import { GetUser } from '@/common/decorator/user.decorator'; import { sessionConfig } from '@/configs/session.config'; +import { StockSearchRequest } from '@/stock/dto/stock.request'; import { StockSearchResponse, StockViewsResponse, @@ -46,7 +47,16 @@ import { UserStockResponse, } from '@/stock/dto/userStock.response'; import { User } from '@/user/domain/user.entity'; -import { StockSearchRequest } from '@/stock/dto/stock.request'; + +const TIME_UNIT = { + MINUTE: 'minute', + DAY: 'day', + WEEK: 'week', + MONTH: 'month', + YEAR: 'year', +} as const; + +type TIME_UNIT = (typeof TIME_UNIT)[keyof typeof TIME_UNIT]; @Controller('stock') export class StockController { @@ -167,61 +177,25 @@ export class StockController { return await this.stockService.searchStock(request.name); } - @Get(':stockId/minutely') - @ApiGetStockData('주식 분 단위 데이터 조회 API', '분') - async getStockDataMinutely( - @Param('stockId') stockId: string, - @Query('lastStartTime') lastStartTime?: string, - ) { - return this.stockDataMinutelyService.getStockDataMinutely( - stockId, - lastStartTime, - ); - } - - @Get(':stockId/daily') - @ApiGetStockData('주식 일 단위 데이터 조회 API', '일') + @Get('/:stockId') + @ApiGetStockData('주식 시간 단위 데이터 조회 API', '일') async getStockDataDaily( @Param('stockId') stockId: string, @Query('lastStartTime') lastStartTime?: string, + @Query('timeunit') timeunit: TIME_UNIT = TIME_UNIT.MINUTE, ) { - return this.stockDataDailyService.getStockDataDaily(stockId, lastStartTime); - } - - @Get(':stockId/weekly') - @ApiGetStockData('주식 주 단위 데이터 조회 API', '주') - async getStockDataWeekly( - @Param('stockId') stockId: string, - @Query('lastStartTime') lastStartTime?: string, - ) { - return this.stockDataWeeklyService.getStockDataWeekly( - stockId, - lastStartTime, - ); - } - - @Get(':stockId/mothly') - @ApiGetStockData('주식 월 단위 데이터 조회 API', '월') - async getStockDataMonthly( - @Param('stockId') stockId: string, - @Query('lastStartTime') lastStartTime?: string, - ) { - return this.stockDataMonthlyService.getStockDataMonthly( - stockId, - lastStartTime, - ); - } - - @Get(':stockId/yearly') - @ApiGetStockData('주식 연 단위 데이터 조회 API', '연') - async getStockDataYearly( - @Param('stockId') stockId: string, - @Query('lastStartTime') lastStartTime?: string, - ) { - return this.stockDataYearlyService.getStockDataYearly( - stockId, - lastStartTime, - ); + switch (timeunit) { + case TIME_UNIT.MINUTE: + return this.getMinutelyData(stockId, lastStartTime); + case TIME_UNIT.DAY: + return this.getDailyData(stockId, lastStartTime); + case TIME_UNIT.MONTH: + return this.getStockDataMonthly(stockId, lastStartTime); + case TIME_UNIT.WEEK: + return this.getStockDataWeekly(stockId, lastStartTime); + default: + return this.getStockDataYearly(stockId, lastStartTime); + } } @ApiOperation({ @@ -257,4 +231,45 @@ export class StockController { async getTopStocksByLosers(@LimitQuery(20) limit: number) { return await this.stockService.getTopStocksByLosers(limit); } + + private getStockDataYearly( + stockId: string, + lastStartTime: string | undefined, + ) { + return this.stockDataYearlyService.getStockDataYearly( + stockId, + lastStartTime, + ); + } + + private getStockDataWeekly( + stockId: string, + lastStartTime: string | undefined, + ) { + return this.stockDataWeeklyService.getStockDataWeekly( + stockId, + lastStartTime, + ); + } + + private getStockDataMonthly( + stockId: string, + lastStartTime: string | undefined, + ) { + return this.stockDataMonthlyService.getStockDataMonthly( + stockId, + lastStartTime, + ); + } + + private getMinutelyData(stockId: string, lastStartTime?: string) { + return this.stockDataMinutelyService.getStockDataMinutely( + stockId, + lastStartTime, + ); + } + + private getDailyData(stockId: string, lastStartTime?: string) { + return this.stockDataDailyService.getStockDataDaily(stockId, lastStartTime); + } } From 46ba8aace1cf1d8967d251e5ba4bdf542566943c Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 21 Nov 2024 15:56:24 +0900 Subject: [PATCH 054/112] =?UTF-8?q?=F0=9F=92=84=20style:=20=ED=81=B0=20?= =?UTF-8?q?=EB=94=B0=EC=98=B4=ED=91=9C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/decorator/stockData.decorator.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/stock/decorator/stockData.decorator.ts b/packages/backend/src/stock/decorator/stockData.decorator.ts index d48dccc1..4941fa61 100644 --- a/packages/backend/src/stock/decorator/stockData.decorator.ts +++ b/packages/backend/src/stock/decorator/stockData.decorator.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable max-lines-per-function */ -import { applyDecorators } from "@nestjs/common"; -import { ApiOperation, ApiParam, ApiQuery, ApiResponse } from "@nestjs/swagger"; -import { StockDataResponse } from "../dto/stockData.response"; +import { applyDecorators } from '@nestjs/common'; +import { ApiOperation, ApiParam, ApiQuery, ApiResponse } from '@nestjs/swagger'; +import { StockDataResponse } from '../dto/stockData.response'; export function ApiGetStockData(summary: string, type: string) { return applyDecorators( From 386f3243d542bc89d50a70f0c3d3e3426a9a9ea1 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 16:10:10 +0900 Subject: [PATCH 055/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20kospi=20stock=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=B6=94=EA=B0=80,=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 40 +++++++++++++++++-- .../src/stock/domain/kospiStock.entity.ts | 3 +- .../backend/src/stock/domain/stock.entity.ts | 2 +- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 7b365780..2ef8f6ae 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -14,6 +14,7 @@ import { import { TR_IDS } from '../type/openapiUtil.type'; import { getOpenApi } from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; +import { KospiStock } from '@/stock/domain/kospiStock.entity'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; import { StockDetail } from '@/stock/domain/stockDetail.entity'; @@ -51,6 +52,12 @@ export class OpenapiDetailData { manager.save(entity, stockDetail); } + private async saveKospiData(stockDetail: KospiStock) { + const manager = this.datasource.manager; + const entity = KospiStock; + manager.save(entity, stockDetail); + } + private async calPer(eps: number): Promise { if (eps <= 0) return NaN; const manager = this.datasource.manager; @@ -99,8 +106,10 @@ export class OpenapiDetailData { private async makeStockDetailObject( output1: FinancialData, output2: ProductDetail, + stockId: string, ): Promise { const result = new StockDetail(); + result.stock = { id: stockId } as Stock; result.marketCap = (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; result.eps = parseInt(output1.eps); @@ -113,10 +122,15 @@ export class OpenapiDetailData { return result; } - private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { - const dataQuery = this.getDetailDataQuery(stock.id!); - const defaultQuery = this.getDefaultDataQuery(stock.id!); + private async makeKospiStockObject(output: ProductDetail, stockId: string) { + const ret = new KospiStock(); + ret.isKospi = output.kospi200_item_yn === 'Y' ? true : false; + ret.stock = { id: stockId } as Stock; + return ret; + } + private async getFinancialData(stock: Stock, conf: typeof openApiConfig) { + const dataQuery = this.getDetailDataQuery(stock.id!); // 여기서 가져올 건 eps -> eps와 per 계산하자. const output1 = await getOpenApi( this.incomeUrl, @@ -124,6 +138,12 @@ export class OpenapiDetailData { dataQuery, TR_IDS.FINANCIAL_DATA, ); + return output1; + } + + private async getProductData(stock: Stock, conf: typeof openApiConfig) { + const defaultQuery = this.getDefaultDataQuery(stock.id!); + // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 const output2 = await getOpenApi( this.defaultUrl, @@ -131,10 +151,22 @@ export class OpenapiDetailData { defaultQuery, TR_IDS.PRODUCTION_DETAIL, ); + return output2; + } + + private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { + const output1 = await this.getFinancialData(stock, conf); + const output2 = await this.getProductData(stock, conf); if (isFinancialData(output1) && isProductDetail(output2)) { - const stockDetail = await this.makeStockDetailObject(output1, output2); + const stockDetail = await this.makeStockDetailObject( + output1, + output2, + stock.id!, + ); this.saveDetailData(stockDetail); + const kospiStock = await this.makeKospiStockObject(output2, stock.id!); + this.saveKospiData(kospiStock); } } diff --git a/packages/backend/src/stock/domain/kospiStock.entity.ts b/packages/backend/src/stock/domain/kospiStock.entity.ts index 7d96a992..8f45a87c 100644 --- a/packages/backend/src/stock/domain/kospiStock.entity.ts +++ b/packages/backend/src/stock/domain/kospiStock.entity.ts @@ -1,4 +1,4 @@ -import { Column, Entity, OneToOne, PrimaryColumn } from 'typeorm'; +import { Column, Entity, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; import { Stock } from './stock.entity'; @Entity() @@ -10,5 +10,6 @@ export class KospiStock { isKospi: boolean; @OneToOne(() => Stock, (stock) => stock.id) + @JoinColumn({ name: 'stock_id' }) stock: Stock; } diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index 66dfb583..be321f55 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -1,4 +1,5 @@ import { Column, Entity, OneToMany, OneToOne, PrimaryColumn } from 'typeorm'; +import { KospiStock } from './kospiStock.entity'; import { StockDaily, StockMinutely, @@ -9,7 +10,6 @@ import { import { Like } from '@/chat/domain/like.entity'; import { DateEmbedded } from '@/common/dateEmbedded.entity'; import { UserStock } from '@/stock/domain/userStock.entity'; -import { KospiStock } from './kospiStock.entity'; @Entity() export class Stock { From 15549267506b82e0416107b150a522a0ab4134d1 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 21 Nov 2024 18:08:51 +0900 Subject: [PATCH 056/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=BF=A0=ED=82=A4?= =?UTF-8?q?=20sameSite=20=EC=98=B5=EC=85=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/session.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/configs/session.config.ts b/packages/backend/src/configs/session.config.ts index 65c49b96..456bda54 100644 --- a/packages/backend/src/configs/session.config.ts +++ b/packages/backend/src/configs/session.config.ts @@ -8,6 +8,8 @@ export const sessionConfig = { resave: false, saveUninitialized: false, name: process.env.COOKIE_NAME, + secure: true, + sameSite: 'none', cookie: { maxAge: Number(process.env.COOKIE_MAX_AGE), }, From d4abdcbcc16447b21f53d094638512cc1338ac3a Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 21 Nov 2024 19:03:37 +0900 Subject: [PATCH 057/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=BF=A0=ED=82=A4?= =?UTF-8?q?=20sameSite=20=EC=98=B5=EC=85=98=20=EC=9E=AC=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/session.config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/configs/session.config.ts b/packages/backend/src/configs/session.config.ts index 456bda54..310aae4d 100644 --- a/packages/backend/src/configs/session.config.ts +++ b/packages/backend/src/configs/session.config.ts @@ -2,15 +2,15 @@ import { randomUUID } from 'node:crypto'; import * as dotenv from 'dotenv'; dotenv.config(); - +type none = 'none'; export const sessionConfig = { secret: process.env.COOKIE_SECRET || randomUUID().toString(), resave: false, saveUninitialized: false, name: process.env.COOKIE_NAME, - secure: true, - sameSite: 'none', cookie: { maxAge: Number(process.env.COOKIE_MAX_AGE), + secure: true, + sameSite: 'none' as none, }, }; From 8e7e73c228673ea2c2b202fa2a8606d9a2be421a Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 19:04:17 +0900 Subject: [PATCH 058/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detail=20?= =?UTF-8?q?=ED=95=AD=EB=AA=A9=20=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 169 ++++++++++++------ .../openapi/type/openapiDetailData.type.ts | 6 +- .../scraper/openapi/type/openapiUtil.type.ts | 4 +- .../scraper/openapi/util/openapiUtil.api.ts | 12 ++ .../openapi/websocketClient.service.ts | 4 +- 5 files changed, 135 insertions(+), 60 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 2ef8f6ae..add3caec 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -1,12 +1,13 @@ -import { Injectable, UseFilters } from '@nestjs/common'; +import { Inject, Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; -import { Between, DataSource } from 'typeorm'; +import { DataSource } from 'typeorm'; +import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { DetailDataQuery, - FinancialData, - isFinancialData, + FinancialRatio, + isFinancialRatioData, isProductDetail, ProductDetail, StockDetailQuery, @@ -23,39 +24,69 @@ import { StockDetail } from '@/stock/domain/stockDetail.entity'; export class OpenapiDetailData { private readonly financialUrl: string = '/uapi/domestic-stock/v1/finance/financial-ratio'; - private readonly defaultUrl: string = + private readonly productUrl: string = '/uapi/domestic-stock/v1/quotations/search-stock-info'; - private readonly incomeUrl: string = - '/uapi/domestic-stock/v1/finance/income-statement'; - private readonly intervals = 100; - private readonly config: (typeof openApiConfig)[] = openApiToken.configs; - constructor(private readonly datasource: DataSource) {} + private readonly intervals = 1000; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + setTimeout(() => this.getDetailData(), 5000); + } @Cron('0 8 * * 1-5') @UseFilters(OpenapiExceptionFilter) public async getDetailData() { - if (process.env.NODE_ENV !== 'production') return; + //if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); - const configCount = this.config.length; + const configCount = openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { + this.logger.info(openApiToken.configs[i]); const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, this.config[i]); + this.getDetailDataChunk(chunk, openApiToken.configs[i]); } } private async saveDetailData(stockDetail: StockDetail) { const manager = this.datasource.manager; const entity = StockDetail; - manager.save(entity, stockDetail); + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } } private async saveKospiData(stockDetail: KospiStock) { const manager = this.datasource.manager; const entity = KospiStock; - manager.save(entity, stockDetail); + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } } private async calPer(eps: number): Promise { @@ -66,10 +97,16 @@ export class OpenapiDetailData { take: 1, order: { createdAt: 'desc' }, }); - const currentPrice = latestResult[0].close; - const per = currentPrice / eps; + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const per = currentPrice / eps; - return per; + if (isNaN(per)) return 0; + else return per; + } else { + return 0; + } } private async calMarketCap(lstg: number) { @@ -79,32 +116,41 @@ export class OpenapiDetailData { take: 1, order: { createdAt: 'desc' }, }); - const currentPrice = latestResult[0].close; - const marketCap = lstg * currentPrice; - return marketCap; + + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const marketCap = lstg * currentPrice; + + if (isNaN(marketCap)) return 0; + else return marketCap; + } else { + return 0; + } } private async get52WeeksLowHigh() { - const manager = this.datasource.manager; - const nowDate = new Date(); - const weeksAgoDate = this.getDate52WeeksAgo(); - // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 - const output = await manager.find(StockDaily, { - select: ['low', 'high'], - where: { - startTime: Between(weeksAgoDate, nowDate), - }, - }); - const result = output.reduce((prev, cur) => { - if (prev.low > cur.low) prev.low = cur.low; - if (prev.high < cur.high) prev.high = cur.high; - return cur; - }, new StockDaily()); - return { low: result.low, high: result.high }; + //const manager = this.datasource.manager; + //const nowDate = new Date(); + //const weeksAgoDate = this.getDate52WeeksAgo(); + //// 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 + //const output = await manager.find(StockDaily, { + // select: ['low', 'high'], + // where: { + // startTime: Between(weeksAgoDate, nowDate), + // }, + //}); + //const result = output.reduce((prev, cur) => { + // if (prev.low > cur.low) prev.low = cur.low; + // if (prev.high < cur.high) prev.high = cur.high; + // return cur; + //}, new StockDaily()); + //return { low: result.low, high: result.high }; + return { low: 0, high: 0 }; } private async makeStockDetailObject( - output1: FinancialData, + output1: FinancialRatio, output2: ProductDetail, stockId: string, ): Promise { @@ -116,8 +162,12 @@ export class OpenapiDetailData { const { low, high } = await this.get52WeeksLowHigh(); result.low52w = low; result.high52w = high; - result.eps = parseInt(output1.eps); - result.per = await this.calPer(parseInt(output1.eps)); + const eps = parseInt(output1.eps); + if (isNaN(eps)) result.eps = 0; + else result.eps = eps; + const per = await this.calPer(eps); + if (isNaN(per)) result.per = 0; + else result.per = per; result.updatedAt = new Date(); return result; } @@ -129,36 +179,45 @@ export class OpenapiDetailData { return ret; } - private async getFinancialData(stock: Stock, conf: typeof openApiConfig) { + private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { const dataQuery = this.getDetailDataQuery(stock.id!); // 여기서 가져올 건 eps -> eps와 per 계산하자. - const output1 = await getOpenApi( - this.incomeUrl, + const response = await getOpenApi( + this.financialUrl, conf, dataQuery, TR_IDS.FINANCIAL_DATA, ); - return output1; + if (response.output) { + const output1 = response.output; + return output1[0]; + } } private async getProductData(stock: Stock, conf: typeof openApiConfig) { - const defaultQuery = this.getDefaultDataQuery(stock.id!); + const defaultQuery = this.getFinancialDataQuery(stock.id!); // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 - const output2 = await getOpenApi( - this.defaultUrl, + const response = await getOpenApi( + this.productUrl, conf, defaultQuery, TR_IDS.PRODUCTION_DETAIL, ); - return output2; + if (response.output) { + const output2 = response.output; + return output2; + //return bufferToObject(output2); + } } private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { - const output1 = await this.getFinancialData(stock, conf); + const output1 = await this.getFinancialRatio(stock, conf); const output2 = await this.getProductData(stock, conf); - if (isFinancialData(output1) && isProductDetail(output2)) { + this.logger.info(JSON.stringify(output1)); + this.logger.info(JSON.stringify(output2)); + if (isFinancialRatioData(output1) && isProductDetail(output2)) { const stockDetail = await this.makeStockDetailObject( output1, output2, @@ -167,6 +226,8 @@ export class OpenapiDetailData { this.saveDetailData(stockDetail); const kospiStock = await this.makeKospiStockObject(output2, stock.id!); this.saveKospiData(kospiStock); + + this.logger.info(`${stock.id!} is saved`); } } @@ -178,25 +239,25 @@ export class OpenapiDetailData { } } - private getDefaultDataQuery( + private getFinancialDataQuery( stockId: string, code: '300' | '301' | '302' | '306' = '300', ): StockDetailQuery { return { pdno: stockId, - code: code, + prdt_type_cd: code, }; } private getDetailDataQuery( stockId: string, divCode: 'J' = 'J', - classify: '0' | '1' = '1', + classify: '0' | '1' = '0', ): DetailDataQuery { return { + fid_div_cls_code: classify, fid_cond_mrkt_div_code: divCode, fid_input_iscd: stockId, - fid_div_cls_code: classify, }; } diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts index 772d8952..38015d48 100644 --- a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -7,7 +7,7 @@ export type DetailDataQuery = { fid_div_cls_code: '0' | '1'; }; -export type FinancialData = { +export type FinancialRatio = { stac_yymm: string; // 결산 년월 grs: string; // 매출액 증가율 bsop_prfi_inrt: string; // 영업 이익 증가율 @@ -20,7 +20,7 @@ export type FinancialData = { lblt_rate: string; // 부채 비율 }; -export function isFinancialData(data: any): data is FinancialData { +export function isFinancialRatioData(data: any): data is FinancialRatio { return ( data && typeof data.stac_yymm === 'string' && @@ -176,7 +176,7 @@ export const isProductDetail = (data: any): data is ProductDetail => { export type StockDetailQuery = { pdno: string; - code: string; + prdt_type_cd: string; }; //export type FinancialDetail = { diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts index bd2e3edd..6df0ca19 100644 --- a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts @@ -1,13 +1,13 @@ export type TR_ID = | 'FHKST03010100' | 'FHKST03010200' - | 'FHKST66430200' + | 'FHKST66430300' | 'HHKDB669107C0' | 'CTPF1002R'; export const TR_IDS: Record = { ITEM_CHART_PRICE: 'FHKST03010100', MINUTE_DATA: 'FHKST03010200', - FINANCIAL_DATA: 'FHKST66430200', + FINANCIAL_DATA: 'FHKST66430300', PRODUCTION_DETAIL: 'CTPF1002R', }; diff --git a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts index 4ec0b29d..fa8f75b4 100644 --- a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts +++ b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts @@ -78,6 +78,7 @@ const getCurrentTime = () => { const seconds = String(now.getSeconds()).padStart(2, '0'); return `${hours}${minutes}${seconds}`; }; + const decryptAES256 = ( encryptedText: string, key: string, @@ -93,6 +94,16 @@ const decryptAES256 = ( return decrypted; }; +const bufferToObject = (buffer: Buffer): any => { + try { + const jsonString = buffer.toString('utf-8'); + return JSON.parse(jsonString); + } catch (error) { + console.error('Failed to convert buffer to object:', error); + throw error; + } +}; + export { postOpenApi, getOpenApi, @@ -100,4 +111,5 @@ export { getPreviousDate, getCurrentTime, decryptAES256, + bufferToObject, }; diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 47bafad1..1e0d6124 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -16,7 +16,9 @@ export class WebsocketClient { @Inject('winston') private readonly logger: Logger, private readonly openapiLiveData: OpenapiLiveData, ) { - this.connect(); + if (process.env.NODE_ENV === 'production') { + this.connect(); + } } // TODO : subscribe 구조로 리팩토링 From b92e3e6830bfabbda174abf7660f148982b7fbe9 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 19:09:04 +0900 Subject: [PATCH 059/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detail=20NaN=20?= =?UTF-8?q?=EC=9E=84=EC=8B=9C=20=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index add3caec..7420e5de 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -1,6 +1,6 @@ import { Inject, Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; -import { DataSource } from 'typeorm'; +import { Between, DataSource } from 'typeorm'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; @@ -130,23 +130,26 @@ export class OpenapiDetailData { } private async get52WeeksLowHigh() { - //const manager = this.datasource.manager; - //const nowDate = new Date(); - //const weeksAgoDate = this.getDate52WeeksAgo(); - //// 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 - //const output = await manager.find(StockDaily, { - // select: ['low', 'high'], - // where: { - // startTime: Between(weeksAgoDate, nowDate), - // }, - //}); - //const result = output.reduce((prev, cur) => { - // if (prev.low > cur.low) prev.low = cur.low; - // if (prev.high < cur.high) prev.high = cur.high; - // return cur; - //}, new StockDaily()); - //return { low: result.low, high: result.high }; - return { low: 0, high: 0 }; + const manager = this.datasource.manager; + const nowDate = new Date(); + const weeksAgoDate = this.getDate52WeeksAgo(); + // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 + const output = await manager.find(StockDaily, { + select: ['low', 'high'], + where: { + startTime: Between(weeksAgoDate, nowDate), + }, + }); + const result = output.reduce((prev, cur) => { + if (prev.low > cur.low) prev.low = cur.low; + if (prev.high < cur.high) prev.high = cur.high; + return cur; + }, new StockDaily()); + let low = 0; + let high = 0; + if (result.low && !isNaN(result.low)) low = result.low; + if (result.high && !isNaN(result.high)) high = result.high; + return { low, high }; } private async makeStockDetailObject( From a393cbb6fcf25350387cf425d3ce5a76a3092fd6 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 20:46:55 +0900 Subject: [PATCH 060/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20type=20ws=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 26da1fd9..a5e45463 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -45,7 +45,8 @@ "typeorm": "^0.3.20", "unzipper": "^0.12.3", "winston": "^3.17.0", - "winston-daily-rotate-file": "^5.0.0" + "winston-daily-rotate-file": "^5.0.0", + "ws": "^8.18.0" }, "devDependencies": { "@nestjs/cli": "^10.0.0", @@ -58,6 +59,7 @@ "@types/passport-google-oauth20": "^2.0.16", "@types/supertest": "^6.0.0", "@types/unzipper": "^0.10.10", + "@types/ws": "^8.5.13", "@typescript-eslint/eslint-plugin": "^8.0.0", "@typescript-eslint/parser": "^8.0.0", "cz-emoji-conventional": "^1.1.0", From 85d8ae025974769a9bdbffa000672b4ca94e9d4b Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 21:14:50 +0900 Subject: [PATCH 061/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20try=20catch=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 7420e5de..57b007a3 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -31,13 +31,13 @@ export class OpenapiDetailData { private readonly datasource: DataSource, @Inject('winston') private readonly logger: Logger, ) { - setTimeout(() => this.getDetailData(), 5000); + //setTimeout(() => this.getDetailData(), 5000); } @Cron('0 8 * * 1-5') @UseFilters(OpenapiExceptionFilter) public async getDetailData() { - //if (process.env.NODE_ENV !== 'production') return; + if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); const configCount = openApiToken.configs.length; @@ -185,15 +185,19 @@ export class OpenapiDetailData { private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { const dataQuery = this.getDetailDataQuery(stock.id!); // 여기서 가져올 건 eps -> eps와 per 계산하자. - const response = await getOpenApi( - this.financialUrl, - conf, - dataQuery, - TR_IDS.FINANCIAL_DATA, - ); - if (response.output) { - const output1 = response.output; - return output1[0]; + try { + const response = await getOpenApi( + this.financialUrl, + conf, + dataQuery, + TR_IDS.FINANCIAL_DATA, + ); + if (response.output) { + const output1 = response.output; + return output1[0]; + } + } catch (error) { + this.logger.error(error); } } @@ -201,16 +205,20 @@ export class OpenapiDetailData { const defaultQuery = this.getFinancialDataQuery(stock.id!); // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 - const response = await getOpenApi( - this.productUrl, - conf, - defaultQuery, - TR_IDS.PRODUCTION_DETAIL, - ); - if (response.output) { - const output2 = response.output; - return output2; - //return bufferToObject(output2); + try { + const response = await getOpenApi( + this.productUrl, + conf, + defaultQuery, + TR_IDS.PRODUCTION_DETAIL, + ); + if (response.output) { + const output2 = response.output; + return output2; + //return bufferToObject(output2); + } + } catch (error) { + this.logger.error(error); } } From e46e1039e5a3dc634c6abe5dab94b999f91291b3 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Thu, 21 Nov 2024 21:19:45 +0900 Subject: [PATCH 062/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20=EC=BB=A4?= =?UTF-8?q?=EC=8A=A4=ED=85=80=20=ED=95=84=ED=84=B0=EC=97=90=20try-catch=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiPeriodData.api.ts | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 5e35f3b9..64db1d9b 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -1,6 +1,7 @@ -import { Injectable, UseFilters } from '@nestjs/common'; +import { Inject, Injectable, UseFilters } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DataSource, EntityManager } from 'typeorm'; +import { Logger } from 'winston'; import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { ChartData, @@ -44,7 +45,10 @@ const INTERVALS = 4000; export class OpenapiPeriodData { private readonly url: string = '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; - public constructor(private readonly datasource: DataSource) { + public constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { //this.getItemChartPriceCheck(); } @@ -123,17 +127,18 @@ export class OpenapiPeriodData { ); } - private async fetchChartData( - query: ItemChartPriceQuery, - configIdx: number, - ): Promise { - const response = await getOpenApi( - this.url, - openApiToken.configs[configIdx], - query, - TR_IDS.ITEM_CHART_PRICE, - ); - return response.output2 as ChartData[]; + private async fetchChartData(query: ItemChartPriceQuery, configIdx: number) { + try { + const response = await getOpenApi( + this.url, + openApiToken.configs[configIdx], + query, + TR_IDS.ITEM_CHART_PRICE, + ); + return response.output2 as ChartData[]; + } catch (error) { + this.logger.error(error); + } } private updateDates( From 4a1ad278ea138f649334ce9b7cac1b9a358519f3 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Thu, 21 Nov 2024 22:51:08 +0900 Subject: [PATCH 063/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=BF=A0=ED=82=A4?= =?UTF-8?q?=20sameSite=20=EC=98=B5=EC=85=98=20=EC=9E=AC=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/session.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/configs/session.config.ts b/packages/backend/src/configs/session.config.ts index 310aae4d..25910f44 100644 --- a/packages/backend/src/configs/session.config.ts +++ b/packages/backend/src/configs/session.config.ts @@ -10,7 +10,9 @@ export const sessionConfig = { name: process.env.COOKIE_NAME, cookie: { maxAge: Number(process.env.COOKIE_MAX_AGE), + httpOnly: true, secure: true, + domain: 'juchum.info', sameSite: 'none' as none, }, }; From bf741722ba994ca405a96736e7526a9577e0108d Mon Sep 17 00:00:00 2001 From: kimminsu Date: Fri, 22 Nov 2024 00:12:55 +0900 Subject: [PATCH 064/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=B1=84=ED=8C=85?= =?UTF-8?q?=20=EC=82=AC=EC=9A=A9=EC=9E=90=20=EB=8B=89=EB=84=A4=EC=9E=84=20?= =?UTF-8?q?=EC=A0=95=EB=B3=B4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/chat/chat.service.ts | 1 + packages/backend/src/chat/dto/chat.response.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/backend/src/chat/chat.service.ts b/packages/backend/src/chat/chat.service.ts index fbb5ac45..80417656 100644 --- a/packages/backend/src/chat/chat.service.ts +++ b/packages/backend/src/chat/chat.service.ts @@ -110,6 +110,7 @@ export class ChatService { .leftJoinAndSelect('chat.likes', 'like', 'like.user_id = :userId', { userId, }) + .leftJoinAndSelect('chat.user', 'user') .where('chat.stock_id = :stockId', { stockId }) .take(size + 1); diff --git a/packages/backend/src/chat/dto/chat.response.ts b/packages/backend/src/chat/dto/chat.response.ts index 803b81e6..bf42c903 100644 --- a/packages/backend/src/chat/dto/chat.response.ts +++ b/packages/backend/src/chat/dto/chat.response.ts @@ -8,6 +8,7 @@ interface ChatResponse { message: string; type: string; liked: boolean; + nickname: string; createdAt: Date; } @@ -25,6 +26,7 @@ export class ChatScrollResponse { id: 1, likeCount: 0, message: '안녕하세요', + nickname: '초보 주주', type: ChatType.NORMAL, isLiked: true, createdAt: new Date(), @@ -41,6 +43,7 @@ export class ChatScrollResponse { type: chat.type, createdAt: chat.date!.createdAt, liked: !!(chat.likes && chat.likes.length > 0), + nickname: chat.user.nickname, })); this.hasMore = hasMore; } From 9e978586d488d51446752a3abc6c6f1f1a166b91 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Fri, 22 Nov 2024 00:29:52 +0900 Subject: [PATCH 065/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=BF=A0=ED=82=A4?= =?UTF-8?q?=20=EC=98=B5=EC=85=98=20=EC=B4=88=EA=B8=B0=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/session.config.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/backend/src/configs/session.config.ts b/packages/backend/src/configs/session.config.ts index 25910f44..6cf984cd 100644 --- a/packages/backend/src/configs/session.config.ts +++ b/packages/backend/src/configs/session.config.ts @@ -2,7 +2,6 @@ import { randomUUID } from 'node:crypto'; import * as dotenv from 'dotenv'; dotenv.config(); -type none = 'none'; export const sessionConfig = { secret: process.env.COOKIE_SECRET || randomUUID().toString(), resave: false, @@ -11,8 +10,5 @@ export const sessionConfig = { cookie: { maxAge: Number(process.env.COOKIE_MAX_AGE), httpOnly: true, - secure: true, - domain: 'juchum.info', - sameSite: 'none' as none, }, }; From e1c702a49c3d051b564a8b8f02781a02bb704a07 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Fri, 22 Nov 2024 14:28:08 +0900 Subject: [PATCH 066/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20websocket=20?= =?UTF-8?q?=EC=9E=84=EC=8B=9C=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .remote/version02/docker-compose.yml | 55 +++++++++++ .remote/version02/frontend.nginx | 36 +++++++ .remote/version02/reverse-proxy.nginx | 77 +++++++++++++++ .remote/version03/docker-compose.yml | 0 .remote/version03/new.conf | 36 +++++++ .remote/version03/reverseproxy.nginx | 86 +++++++++++++++++ .remote/version04/docker-compose.yml | 43 +++++++++ .remote/version04/frontend.nginx | 86 +++++++++++++++++ .remote/version04/new-reverse_proxy.nginx | 93 +++++++++++++++++++ .remote/version04/newfrontend.nginx | 73 +++++++++++++++ .remote/version04/reverse-proxy.nginx | 86 +++++++++++++++++ .remote/version05/docker-compose.yml | 42 +++++++++ .remote/version05/frontend.nginx | 80 ++++++++++++++++ packages/backend/package.json | 3 +- .../openapi/api/openapiLiveData.api.ts | 13 ++- .../scraper/openapi/api/openapiToken.api.ts | 1 + .../openapi/websocketClient.service.ts | 13 ++- 17 files changed, 813 insertions(+), 10 deletions(-) create mode 100644 .remote/version02/docker-compose.yml create mode 100644 .remote/version02/frontend.nginx create mode 100644 .remote/version02/reverse-proxy.nginx create mode 100644 .remote/version03/docker-compose.yml create mode 100644 .remote/version03/new.conf create mode 100644 .remote/version03/reverseproxy.nginx create mode 100644 .remote/version04/docker-compose.yml create mode 100644 .remote/version04/frontend.nginx create mode 100644 .remote/version04/new-reverse_proxy.nginx create mode 100644 .remote/version04/newfrontend.nginx create mode 100644 .remote/version04/reverse-proxy.nginx create mode 100644 .remote/version05/docker-compose.yml create mode 100644 .remote/version05/frontend.nginx diff --git a/.remote/version02/docker-compose.yml b/.remote/version02/docker-compose.yml new file mode 100644 index 00000000..32fa5759 --- /dev/null +++ b/.remote/version02/docker-compose.yml @@ -0,0 +1,55 @@ +networks: + corp: + driver: bridge + +services: + nginx_proxy: + image: nginx:1.27.2-alpine + container_name: nginx_proxy + ports: + - '80:80' + - '443:443' + depends_on: #proxy가 먼저 켜지는 경우 죽어버리는 문제가 있었음. 의존성 추가 + - frontend + - backend + volumes: + - ./nginx/reverse-proxy.conf:/etc/nginx/nginx.conf + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'" + networks: + - corp + + certbot: + image: certbot/certbot + volumes: + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" + + frontend: + image: sunghwki/frontend:latest + env_file: + - .env + environment: + - NODE_ENV=production + depends_on: + - backend + volumes: + - ./nginx/frontend.conf:/etc/nginx/nginx.conf + networks: + - corp + + backend: + image: sunghwki/backend:latest + env_file: + - .env + environment: + - DATABASE_URL=mysql://${DB_USER}:${DB_PASS}@db:${DB_PORT}/${DB_NAME} + volumes: + - ./logs:/packages/packages/logs + networks: + - corp + +volumes: + db-data: diff --git a/.remote/version02/frontend.nginx b/.remote/version02/frontend.nginx new file mode 100644 index 00000000..f26a67ed --- /dev/null +++ b/.remote/version02/frontend.nginx @@ -0,0 +1,36 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 8080; + + location / { + root /usr/share/nginx/html; + try_files $uri /index.html; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; # 이게 문제였다. 여기서 80포트를 포함시켜서 이미 있는 포트를 점유하는 중이었고, 그러다보니 문제가 발생했다. +} diff --git a/.remote/version02/reverse-proxy.nginx b/.remote/version02/reverse-proxy.nginx new file mode 100644 index 00000000..9887d1fd --- /dev/null +++ b/.remote/version02/reverse-proxy.nginx @@ -0,0 +1,77 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + server { + listen 80; + server_name juchum.info; + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + location / { + return 301 https://$host$request_uri; + } + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + proxy_pass http://static-server/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version03/docker-compose.yml b/.remote/version03/docker-compose.yml new file mode 100644 index 00000000..e69de29b diff --git a/.remote/version03/new.conf b/.remote/version03/new.conf new file mode 100644 index 00000000..f26a67ed --- /dev/null +++ b/.remote/version03/new.conf @@ -0,0 +1,36 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 8080; + + location / { + root /usr/share/nginx/html; + try_files $uri /index.html; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; # 이게 문제였다. 여기서 80포트를 포함시켜서 이미 있는 포트를 점유하는 중이었고, 그러다보니 문제가 발생했다. +} diff --git a/.remote/version03/reverseproxy.nginx b/.remote/version03/reverseproxy.nginx new file mode 100644 index 00000000..2be13402 --- /dev/null +++ b/.remote/version03/reverseproxy.nginx @@ -0,0 +1,86 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 80; + server_name juchum.info; + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + proxy_pass http://static-server/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version04/docker-compose.yml b/.remote/version04/docker-compose.yml new file mode 100644 index 00000000..0feca385 --- /dev/null +++ b/.remote/version04/docker-compose.yml @@ -0,0 +1,43 @@ +networks: + corp: + driver: bridge + +services: + + frontend: + image: sunghwki/frontend:latest + env_file: + - .env + environment: + - NODE_ENV=production + depends_on: + - backend + volumes: + - ./nginx/frontend.conf:/etc/nginx/nginx.conf + - ./nginx/reverse-proxy.conf:/etc/nginx/nginx.conf + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + command: '/bin/sh -c ''while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g "daemon off;"''' + networks: + - corp + + certbot: + image: certbot/certbot + volumes: + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" + + backend: + image: sunghwki/backend:latest + env_file: + - .env + environment: + - DATABASE_URL=mysql://${DB_USER}:${DB_PASS}@db:${DB_PORT}/${DB_NAME} + volumes: + - ./logs:/packages/packages/logs + networks: + - corp + +volumes: + db-data: diff --git a/.remote/version04/frontend.nginx b/.remote/version04/frontend.nginx new file mode 100644 index 00000000..2be13402 --- /dev/null +++ b/.remote/version04/frontend.nginx @@ -0,0 +1,86 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 80; + server_name juchum.info; + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + proxy_pass http://static-server/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version04/new-reverse_proxy.nginx b/.remote/version04/new-reverse_proxy.nginx new file mode 100644 index 00000000..c75785bf --- /dev/null +++ b/.remote/version04/new-reverse_proxy.nginx @@ -0,0 +1,93 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 80; + server_name juchum.info; + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + proxy_pass http://static-server/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://juchum.info + proxy_redirect off; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version04/newfrontend.nginx b/.remote/version04/newfrontend.nginx new file mode 100644 index 00000000..e3a75870 --- /dev/null +++ b/.remote/version04/newfrontend.nginx @@ -0,0 +1,73 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + server { + listen 80; + server_name juchum.info; + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + root /usr/share/nginx/html; + index index.html; + try_files $uri $uri/ /index.html; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version04/reverse-proxy.nginx b/.remote/version04/reverse-proxy.nginx new file mode 100644 index 00000000..2be13402 --- /dev/null +++ b/.remote/version04/reverse-proxy.nginx @@ -0,0 +1,86 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + # 프론트엔드 upstream 설정 + upstream static-server { + server frontend:8080; + } + + server { + listen 80; + server_name juchum.info; + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + proxy_pass http://static-server/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/.remote/version05/docker-compose.yml b/.remote/version05/docker-compose.yml new file mode 100644 index 00000000..68a36250 --- /dev/null +++ b/.remote/version05/docker-compose.yml @@ -0,0 +1,42 @@ +networks: + corp: + driver: bridge +services: + frontend: + image: sunghwki/frontend:latest + build: + context: ./frontend + dockerfile: Dockerfile + ports: + - "80:80" + - "443:443" + env_file: + - .env + depends_on: + - backend + volumes: + - ./nginx/frontend.conf:/etc/nginx/nginx.conf + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + command: '/bin/sh -c ''while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g "daemon off;"''' + networks: + - corp + + certbot: + image: certbot/certbot + volumes: + - ./data/certbot/conf:/etc/letsencrypt + - ./data/certbot/www:/var/www/certbot + entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" + + backend: + image: sunghwki/backend:latest + env_file: + - .env + volumes: + - ./logs:/packages/packages/logs + networks: + - corp + +volumes: + db-data: diff --git a/.remote/version05/frontend.nginx b/.remote/version05/frontend.nginx new file mode 100644 index 00000000..7d343da3 --- /dev/null +++ b/.remote/version05/frontend.nginx @@ -0,0 +1,80 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # 백엔드 upstream 설정 + upstream nest-api-server { + server backend:3000; + } + + server { + listen 80; + server_name juchum.info; + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + } + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + } + + server { + listen 443 ssl; + server_name juchum.info; + + ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; + + location / { + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_pass http://nest-api-server/api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_cookie_domain nest-api-server juchum.info; + proxy_cookie_path / /; + } + + location /socket.io { + proxy_pass http://nest-api-server/socket.io; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + #include /etc/nginx/conf.d/*.conf; +} diff --git a/packages/backend/package.json b/packages/backend/package.json index a5e45463..5db955b3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -17,7 +17,8 @@ "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config ./test/jest-e2e.json" + "test:e2e": "jest --config ./test/jest-e2e.json", + "mem": "node ../../dist/main --inspect" }, "dependencies": { "@nestjs/common": "^10.0.0", diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 77a31c67..456cff90 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -57,7 +57,11 @@ export class OpenapiLiveData { } private async getKospiStockId() { - const kospi = await this.manager.find(KospiStock); + const kospi = await this.manager.find(KospiStock, { + where: { + isKospi: true, + }, + }); return kospi; } @@ -82,12 +86,13 @@ export class OpenapiLiveData { public async output(message: Buffer, iv?: string, key?: string) { const parsed = message.toString().split('|'); + console.log(message.toString()); if (parsed.length > 0) { if (parsed[0] == '1' && iv && key) - parsed[4] = decryptAES256(parsed[4], iv, key); + parsed[3] = decryptAES256(parsed[3], iv, key); if (parsed[1] !== this.TR_ID) return; - const stockData = parsed[4].split('^'); - const length = stockData.length / parseInt(parsed[3]); + const stockData = parsed[3].split('^'); + const length = stockData.length / parseInt(parsed[2]); const size = parseInt(parsed[2]); const i = 0; while (i < size) { diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 275730bc..fa4901a8 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -32,6 +32,7 @@ class OpenapiTokenApi { } public get configs() { + //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 return this.config; } diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 1e0d6124..554c3eff 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -26,13 +26,12 @@ export class WebsocketClient { private message(data: any) { this.logger.info(`Received message: ${data}`); - const message = JSON.parse(data); - if (message.header && message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(message)}`); + if (data.header && data.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(data)}`); this.sendPong(); return; } - if (message.header && message.header.tr_id === 'H0STCNT0') { + if (data.header && data.header.tr_id === 'H0STCNT0') { return; } this.openapiLiveData.output(data); @@ -50,7 +49,11 @@ export class WebsocketClient { }); this.client.on('message', (data: any) => { - this.message(data); + try { + this.message(data); + } catch (error) { + this.logger.info(error); + } }); this.client.on('close', () => { From e55cadd6feb1d00b8cd2473112a3bfec9a509c9c Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:46:14 +0900 Subject: [PATCH 067/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EC=9C=A0=EC=A0=80=20=EC=83=9D=EC=84=B1=20=EA=B8=B0?= =?UTF-8?q?=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/user/user.service.ts | 72 +++++++++++++++++++---- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/user/user.service.ts b/packages/backend/src/user/user.service.ts index c1eebd52..8d2a5d23 100644 --- a/packages/backend/src/user/user.service.ts +++ b/packages/backend/src/user/user.service.ts @@ -27,22 +27,25 @@ export class UserService { }); } + async registerTester() { + return await this.dataSource.transaction(async (manager) => { + return await manager.save(User, { + nickname: this.generateRandomNickname(), + email: 'tester@nav', + type: OauthType.LOCAL, + oauthId: String( + (await this.getMaxOauthId(OauthType.LOCAL, manager)) + 1, + ), + }); + }); + } + async findUserByOauthIdAndType(oauthId: string, type: OauthType) { return await this.dataSource.manager.findOne(User, { where: { oauthId, type }, }); } - private async validateUserExists( - type: OauthType, - oauthId: string, - manager: EntityManager, - ) { - if (await manager.exists(User, { where: { oauthId, type } })) { - throw new BadRequestException('user already exists'); - } - } - async updateUserTheme(userId: number, isLight?: boolean): Promise { return await this.dataSource.transaction(async (manager) => { if (isLight === undefined) { @@ -72,4 +75,53 @@ export class UserService { return user.isLight; } + + private generateRandomNickname(): string { + const adjectives = [ + '강력한', + '지혜로운', + '소중한', + '빛나는', + '고요한', + '용감한', + '행운의', + '신비로운', + ]; + const animals = [ + '호랑이', + '독수리', + '용', + '사슴', + '백호', + '하늘새', + '백두산 호랑이', + '붉은 여우', + ]; + + const randomAdjective = + adjectives[Math.floor(Math.random() * adjectives.length)]; + const randomAnimal = animals[Math.floor(Math.random() * animals.length)]; + + return `${randomAdjective} ${randomAnimal}`; + } + + private async getMaxOauthId(oauthType: OauthType, manager: EntityManager) { + const result = await manager + .createQueryBuilder(User, 'user') + .select('MAX(user.oauthId)', 'max') + .where('user.type = :oauthType', { oauthType }) + .getRawOne(); + + return result ? Number(result.max) : 1; + } + + private async validateUserExists( + type: OauthType, + oauthId: string, + manager: EntityManager, + ) { + if (await manager.exists(User, { where: { oauthId, type } })) { + throw new BadRequestException('user already exists'); + } + } } From fc9adbd336bd87efdcb2a1eb72a90b6986d73f9e Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:51:45 +0900 Subject: [PATCH 068/112] =?UTF-8?q?=F0=9F=9A=9A=20chore:=20passport=20loca?= =?UTF-8?q?l=20=ED=8C=A8=ED=82=A4=EC=A7=80=20=EC=84=A4=EC=B9=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/package.json | 2 ++ yarn.lock | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/backend/package.json b/packages/backend/package.json index c41b6046..fd70879e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -39,6 +39,7 @@ "nest-winston": "^1.9.7", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", + "passport-local": "^1.0.0", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1", "socket.io": "^4.8.1", @@ -56,6 +57,7 @@ "@types/jest": "^29.5.2", "@types/node": "^20.3.1", "@types/passport-google-oauth20": "^2.0.16", + "@types/passport-local": "^1.0.38", "@types/supertest": "^6.0.0", "@types/unzipper": "^0.10.10", "@typescript-eslint/eslint-plugin": "^8.0.0", diff --git a/yarn.lock b/yarn.lock index b8b27206..1d75dcf2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2156,6 +2156,15 @@ "@types/passport" "*" "@types/passport-oauth2" "*" +"@types/passport-local@^1.0.38": + version "1.0.38" + resolved "https://registry.yarnpkg.com/@types/passport-local/-/passport-local-1.0.38.tgz#8073758188645dde3515808999b1c218a6fe7141" + integrity sha512-nsrW4A963lYE7lNTv9cr5WmiUD1ibYJvWrpE13oxApFsRt77b0RdtZvKbCdNIY4v/QZ6TRQWaDDEwV1kCTmcXg== + dependencies: + "@types/express" "*" + "@types/passport" "*" + "@types/passport-strategy" "*" + "@types/passport-oauth2@*": version "1.4.17" resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.17.tgz#d5d54339d44f6883d03e69dc0cc0e2114067abb4" @@ -2165,6 +2174,14 @@ "@types/oauth" "*" "@types/passport" "*" +"@types/passport-strategy@*": + version "0.2.38" + resolved "https://registry.yarnpkg.com/@types/passport-strategy/-/passport-strategy-0.2.38.tgz#482abba0b165cd4553ec8b748f30b022bd6c04d3" + integrity sha512-GC6eMqqojOooq993Tmnmp7AUTbbQSgilyvpCYQjT+H6JfG/g6RGc7nXEniZlp0zyKJ0WUdOiZWLBZft9Yug1uA== + dependencies: + "@types/express" "*" + "@types/passport" "*" + "@types/passport@*": version "1.0.17" resolved "https://registry.yarnpkg.com/@types/passport/-/passport-1.0.17.tgz#718a8d1f7000ebcf6bbc0853da1bc8c4bc7ea5e6" @@ -7039,6 +7056,13 @@ passport-google-oauth20@^2.0.0: dependencies: passport-oauth2 "1.x.x" +passport-local@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/passport-local/-/passport-local-1.0.0.tgz#1fe63268c92e75606626437e3b906662c15ba6ee" + integrity sha512-9wCE6qKznvf9mQYYbgJ3sVOHmCWoUNMVFoZzNoznmISbhnNNPhN9xfY3sLmScHMetEJeoY7CXwfhCe7argfQow== + dependencies: + passport-strategy "1.x.x" + passport-oauth2@1.x.x: version "1.8.0" resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.8.0.tgz#55725771d160f09bbb191828d5e3d559eee079c8" From 07d2a8234cf575ae1cac9a290f44ef66ff11b225 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:54:03 +0900 Subject: [PATCH 069/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EC=9C=A0=EC=A0=80=20service=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/auth/tester/testerAuth.service.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 packages/backend/src/auth/tester/testerAuth.service.ts diff --git a/packages/backend/src/auth/tester/testerAuth.service.ts b/packages/backend/src/auth/tester/testerAuth.service.ts new file mode 100644 index 00000000..0982c4b0 --- /dev/null +++ b/packages/backend/src/auth/tester/testerAuth.service.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { UserService } from '@/user/user.service'; + +@Injectable() +export class TesterAuthService { + constructor(private readonly userService: UserService) {} + + async attemptAuthentication() { + return await this.userService.registerTester(); + } +} From 2bc72478280d6c43b8b01e6e931438005830b3cc Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:54:30 +0900 Subject: [PATCH 070/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EC=9C=A0=EC=A0=80=20strategy=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/auth/tester/strategy/tester.strategy.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 packages/backend/src/auth/tester/strategy/tester.strategy.ts diff --git a/packages/backend/src/auth/tester/strategy/tester.strategy.ts b/packages/backend/src/auth/tester/strategy/tester.strategy.ts new file mode 100644 index 00000000..2c6939ca --- /dev/null +++ b/packages/backend/src/auth/tester/strategy/tester.strategy.ts @@ -0,0 +1,16 @@ +import { Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; +import { Strategy } from 'passport-local'; +import { TesterAuthService } from '@/auth/tester/testerAuth.service'; + +@Injectable() +export class TesterStrategy extends PassportStrategy(Strategy) { + constructor(private readonly testerAuthService: TesterAuthService) { + super(); + } + + async validate(username: string, password: string, done: CallableFunction) { + const user = await this.testerAuthService.attemptAuthentication(); + done(null, user); + } +} From 1de6bc2f450faf504f6c20246599a5ccec8cc64d Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:54:45 +0900 Subject: [PATCH 071/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EC=9C=A0=EC=A0=80=20guard=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/auth/tester/guard/tester.guard.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 packages/backend/src/auth/tester/guard/tester.guard.ts diff --git a/packages/backend/src/auth/tester/guard/tester.guard.ts b/packages/backend/src/auth/tester/guard/tester.guard.ts new file mode 100644 index 00000000..46f461d3 --- /dev/null +++ b/packages/backend/src/auth/tester/guard/tester.guard.ts @@ -0,0 +1,16 @@ +import { ExecutionContext, Injectable } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; + +@Injectable() +export class TestAuthGuard extends AuthGuard('local') { + constructor() { + super(); + } + + async canActivate(context: ExecutionContext) { + const isActivate = (await super.canActivate(context)) as boolean; + const request = context.switchToHttp().getRequest(); + await super.logIn(request); + return isActivate; + } +} From 258044beeb15da43e3c68a0c48e653bb9e4fbce0 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:55:04 +0900 Subject: [PATCH 072/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EC=9C=A0=EC=A0=80=20=EB=A1=9C=EA=B7=B8=EC=9D=B8=20?= =?UTF-8?q?=EA=B8=B0=EB=8A=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/auth/auth.module.ts | 16 +++++++-- .../src/auth/tester/testerAuth.controller.ts | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 packages/backend/src/auth/tester/testerAuth.controller.ts diff --git a/packages/backend/src/auth/auth.module.ts b/packages/backend/src/auth/auth.module.ts index f513d613..c32261b7 100644 --- a/packages/backend/src/auth/auth.module.ts +++ b/packages/backend/src/auth/auth.module.ts @@ -1,13 +1,23 @@ import { Module } from '@nestjs/common'; +import { PassportModule } from '@nestjs/passport'; import { GoogleAuthController } from '@/auth/google/googleAuth.controller'; import { GoogleAuthService } from '@/auth/google/googleAuth.service'; import { GoogleStrategy } from '@/auth/google/strategy/google.strategy'; import { SessionSerializer } from '@/auth/session/session.serializer'; +import { TesterStrategy } from '@/auth/tester/strategy/tester.strategy'; +import { TesterAuthController } from '@/auth/tester/testerAuth.controller'; +import { TesterAuthService } from '@/auth/tester/testerAuth.service'; import { UserModule } from '@/user/user.module'; @Module({ - imports: [UserModule], - controllers: [GoogleAuthController], - providers: [GoogleStrategy, GoogleAuthService, SessionSerializer], + imports: [UserModule, PassportModule.register({ session: true })], + controllers: [GoogleAuthController, TesterAuthController], + providers: [ + GoogleStrategy, + GoogleAuthService, + SessionSerializer, + TesterAuthService, + TesterStrategy, + ], }) export class AuthModule {} diff --git a/packages/backend/src/auth/tester/testerAuth.controller.ts b/packages/backend/src/auth/tester/testerAuth.controller.ts new file mode 100644 index 00000000..d4381db9 --- /dev/null +++ b/packages/backend/src/auth/tester/testerAuth.controller.ts @@ -0,0 +1,36 @@ +import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; +import { ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { Request, Response } from 'express'; +import { TestAuthGuard } from '@/auth/tester/guard/tester.guard'; + +@ApiTags('Auth') +@Controller('auth/tester') +export class TesterAuthController { + constructor() {} + + @ApiOperation({ + summary: '테스터 로그인 api', + description: '테스터로 로그인합니다.', + }) + @Get('/login') + @UseGuards(TestAuthGuard) + async handleLogin(@Res() response: Response) { + response.redirect('/'); + } + + @ApiOperation({ + summary: '로그인 상태 확인', + description: '로그인 상태를 확인합니다.', + }) + @ApiOkResponse({ + description: '로그인된 상태', + example: { message: 'Authenticated' }, + }) + @Get('/status') + async user(@Req() request: Request) { + if (request.user) { + return { message: 'Authenticated' }; + } + return { message: 'Not Authenticated' }; + } +} From afb3bf339b857948441ce825eae8e75e03971ce8 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Sat, 23 Nov 2024 21:55:59 +0900 Subject: [PATCH 073/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20google?= =?UTF-8?q?=20strategy=20=EB=8D=94=EC=9D=B4=EC=83=81=20=EC=82=AC=EC=9A=A9?= =?UTF-8?q?=ED=95=98=EC=A7=80=20=EC=95=8A=EB=8A=94=20=EC=9D=98=EC=A1=B4?= =?UTF-8?q?=EC=84=B1=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/auth/google/strategy/google.strategy.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/auth/google/strategy/google.strategy.ts b/packages/backend/src/auth/google/strategy/google.strategy.ts index b87b7945..28295a2d 100644 --- a/packages/backend/src/auth/google/strategy/google.strategy.ts +++ b/packages/backend/src/auth/google/strategy/google.strategy.ts @@ -1,9 +1,8 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { PassportStrategy } from '@nestjs/passport'; import { Profile, Strategy, VerifyCallback } from 'passport-google-oauth20'; import { GoogleAuthService } from '@/auth/google/googleAuth.service'; import { OauthType } from '@/user/domain/ouathType'; -import { Logger } from 'winston'; export interface OauthUserInfo { type: OauthType; @@ -15,7 +14,7 @@ export interface OauthUserInfo { @Injectable() export class GoogleStrategy extends PassportStrategy(Strategy) { - constructor(private readonly googleAuthService: GoogleAuthService, @Inject('winston') private readonly logger: Logger) { + constructor(private readonly googleAuthService: GoogleAuthService) { super({ clientID: process.env.GOOGLE_CLIENT_ID, clientSecret: process.env.GOOGLE_CLIENT_SECRET, From 8cbd2e4a8d665c50d519ab19a8037a8b6a0d936a Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 10:49:42 +0900 Subject: [PATCH 074/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openapiPeriodData?= =?UTF-8?q?=20=EC=88=98=EC=A0=95=20-=20=EB=B6=88=ED=95=84=EC=9A=94?= =?UTF-8?q?=ED=95=9C=20private,=20filter=20=EC=82=AD=EC=A0=9C,=20=EB=A1=9C?= =?UTF-8?q?=EC=A7=81=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + .remote/version02/docker-compose.yml | 55 ---- .remote/version02/frontend.nginx | 36 --- .remote/version02/reverse-proxy.nginx | 77 ----- .remote/version03/docker-compose.yml | 0 .remote/version03/new.conf | 36 --- .remote/version03/reverseproxy.nginx | 86 ------ .remote/version04/docker-compose.yml | 43 --- .remote/version04/frontend.nginx | 86 ------ .remote/version04/new-reverse_proxy.nginx | 93 ------ .remote/version04/newfrontend.nginx | 73 ----- .remote/version04/reverse-proxy.nginx | 86 ------ .remote/version05/docker-compose.yml | 42 --- .remote/version05/frontend.nginx | 80 ----- .../openapi/api/openapiDetailData.api.ts | 280 ++++++++++++++++++ .../openapi/api/openapiLiveData.api.ts | 104 +++++++ .../openapi/api/openapiMinuteData.api.ts | 156 ++++++++++ .../openapi/api/openapiPeriodData.api.ts | 218 ++++++++++++++ .../openapi/api/openapiToken.api.ts | 114 +++++++ .../openapi/config/openapi.config.ts | 17 ++ .../openapi/openapi-scraper.module.ts | 43 +++ .../openapi/openapi-scraper.service.ts | 15 + .../openapi/type/openapiDetailData.type.ts | 214 +++++++++++++ .../openapi/type/openapiLiveData.type.ts | 152 ++++++++++ .../openapi/type/openapiMinuteData.type.ts | 33 +++ .../openapi/type/openapiPeriodData.ts | 45 +++ .../openapi/type/openapiUtil.type.ts | 13 + .../openapi/util/openapiCustom.error.ts | 13 + .../openapi/util/openapiUtil.api.ts | 115 +++++++ .../openapi/websocketClient.service.ts | 87 ++++++ 30 files changed, 1621 insertions(+), 793 deletions(-) delete mode 100644 .remote/version02/docker-compose.yml delete mode 100644 .remote/version02/frontend.nginx delete mode 100644 .remote/version02/reverse-proxy.nginx delete mode 100644 .remote/version03/docker-compose.yml delete mode 100644 .remote/version03/new.conf delete mode 100644 .remote/version03/reverseproxy.nginx delete mode 100644 .remote/version04/docker-compose.yml delete mode 100644 .remote/version04/frontend.nginx delete mode 100644 .remote/version04/new-reverse_proxy.nginx delete mode 100644 .remote/version04/newfrontend.nginx delete mode 100644 .remote/version04/reverse-proxy.nginx delete mode 100644 .remote/version05/docker-compose.yml delete mode 100644 .remote/version05/frontend.nginx create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts diff --git a/.gitignore b/.gitignore index 57004910..bdc0ede0 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # vscode setting .vscode +# remote +.remote diff --git a/.remote/version02/docker-compose.yml b/.remote/version02/docker-compose.yml deleted file mode 100644 index 32fa5759..00000000 --- a/.remote/version02/docker-compose.yml +++ /dev/null @@ -1,55 +0,0 @@ -networks: - corp: - driver: bridge - -services: - nginx_proxy: - image: nginx:1.27.2-alpine - container_name: nginx_proxy - ports: - - '80:80' - - '443:443' - depends_on: #proxy가 먼저 켜지는 경우 죽어버리는 문제가 있었음. 의존성 추가 - - frontend - - backend - volumes: - - ./nginx/reverse-proxy.conf:/etc/nginx/nginx.conf - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'" - networks: - - corp - - certbot: - image: certbot/certbot - volumes: - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" - - frontend: - image: sunghwki/frontend:latest - env_file: - - .env - environment: - - NODE_ENV=production - depends_on: - - backend - volumes: - - ./nginx/frontend.conf:/etc/nginx/nginx.conf - networks: - - corp - - backend: - image: sunghwki/backend:latest - env_file: - - .env - environment: - - DATABASE_URL=mysql://${DB_USER}:${DB_PASS}@db:${DB_PORT}/${DB_NAME} - volumes: - - ./logs:/packages/packages/logs - networks: - - corp - -volumes: - db-data: diff --git a/.remote/version02/frontend.nginx b/.remote/version02/frontend.nginx deleted file mode 100644 index f26a67ed..00000000 --- a/.remote/version02/frontend.nginx +++ /dev/null @@ -1,36 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 8080; - - location / { - root /usr/share/nginx/html; - try_files $uri /index.html; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; # 이게 문제였다. 여기서 80포트를 포함시켜서 이미 있는 포트를 점유하는 중이었고, 그러다보니 문제가 발생했다. -} diff --git a/.remote/version02/reverse-proxy.nginx b/.remote/version02/reverse-proxy.nginx deleted file mode 100644 index 9887d1fd..00000000 --- a/.remote/version02/reverse-proxy.nginx +++ /dev/null @@ -1,77 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - server { - listen 80; - server_name juchum.info; - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - location / { - return 301 https://$host$request_uri; - } - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - proxy_pass http://static-server/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version03/docker-compose.yml b/.remote/version03/docker-compose.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/.remote/version03/new.conf b/.remote/version03/new.conf deleted file mode 100644 index f26a67ed..00000000 --- a/.remote/version03/new.conf +++ /dev/null @@ -1,36 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 8080; - - location / { - root /usr/share/nginx/html; - try_files $uri /index.html; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; # 이게 문제였다. 여기서 80포트를 포함시켜서 이미 있는 포트를 점유하는 중이었고, 그러다보니 문제가 발생했다. -} diff --git a/.remote/version03/reverseproxy.nginx b/.remote/version03/reverseproxy.nginx deleted file mode 100644 index 2be13402..00000000 --- a/.remote/version03/reverseproxy.nginx +++ /dev/null @@ -1,86 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 80; - server_name juchum.info; - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - proxy_pass http://static-server/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version04/docker-compose.yml b/.remote/version04/docker-compose.yml deleted file mode 100644 index 0feca385..00000000 --- a/.remote/version04/docker-compose.yml +++ /dev/null @@ -1,43 +0,0 @@ -networks: - corp: - driver: bridge - -services: - - frontend: - image: sunghwki/frontend:latest - env_file: - - .env - environment: - - NODE_ENV=production - depends_on: - - backend - volumes: - - ./nginx/frontend.conf:/etc/nginx/nginx.conf - - ./nginx/reverse-proxy.conf:/etc/nginx/nginx.conf - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - command: '/bin/sh -c ''while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g "daemon off;"''' - networks: - - corp - - certbot: - image: certbot/certbot - volumes: - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" - - backend: - image: sunghwki/backend:latest - env_file: - - .env - environment: - - DATABASE_URL=mysql://${DB_USER}:${DB_PASS}@db:${DB_PORT}/${DB_NAME} - volumes: - - ./logs:/packages/packages/logs - networks: - - corp - -volumes: - db-data: diff --git a/.remote/version04/frontend.nginx b/.remote/version04/frontend.nginx deleted file mode 100644 index 2be13402..00000000 --- a/.remote/version04/frontend.nginx +++ /dev/null @@ -1,86 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 80; - server_name juchum.info; - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - proxy_pass http://static-server/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version04/new-reverse_proxy.nginx b/.remote/version04/new-reverse_proxy.nginx deleted file mode 100644 index c75785bf..00000000 --- a/.remote/version04/new-reverse_proxy.nginx +++ /dev/null @@ -1,93 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 80; - server_name juchum.info; - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - proxy_pass http://static-server/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Real-IP $remote_addr; - proxy_pass http://juchum.info - proxy_redirect off; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version04/newfrontend.nginx b/.remote/version04/newfrontend.nginx deleted file mode 100644 index e3a75870..00000000 --- a/.remote/version04/newfrontend.nginx +++ /dev/null @@ -1,73 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - server { - listen 80; - server_name juchum.info; - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - root /usr/share/nginx/html; - index index.html; - try_files $uri $uri/ /index.html; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version04/reverse-proxy.nginx b/.remote/version04/reverse-proxy.nginx deleted file mode 100644 index 2be13402..00000000 --- a/.remote/version04/reverse-proxy.nginx +++ /dev/null @@ -1,86 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - # 프론트엔드 upstream 설정 - upstream static-server { - server frontend:8080; - } - - server { - listen 80; - server_name juchum.info; - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - proxy_pass http://static-server/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/.remote/version05/docker-compose.yml b/.remote/version05/docker-compose.yml deleted file mode 100644 index 68a36250..00000000 --- a/.remote/version05/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -networks: - corp: - driver: bridge -services: - frontend: - image: sunghwki/frontend:latest - build: - context: ./frontend - dockerfile: Dockerfile - ports: - - "80:80" - - "443:443" - env_file: - - .env - depends_on: - - backend - volumes: - - ./nginx/frontend.conf:/etc/nginx/nginx.conf - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - command: '/bin/sh -c ''while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g "daemon off;"''' - networks: - - corp - - certbot: - image: certbot/certbot - volumes: - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" - - backend: - image: sunghwki/backend:latest - env_file: - - .env - volumes: - - ./logs:/packages/packages/logs - networks: - - corp - -volumes: - db-data: diff --git a/.remote/version05/frontend.nginx b/.remote/version05/frontend.nginx deleted file mode 100644 index 7d343da3..00000000 --- a/.remote/version05/frontend.nginx +++ /dev/null @@ -1,80 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # 백엔드 upstream 설정 - upstream nest-api-server { - server backend:3000; - } - - server { - listen 80; - server_name juchum.info; - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - } - - location / { - return 301 https://$host$request_uri; - } - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - } - - server { - listen 443 ssl; - server_name juchum.info; - - ssl_certificate /etc/letsencrypt/live/juchum.info/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/juchum.info/privkey.pem; - - location / { - root /usr/share/nginx/html; - try_files $uri /index.html; - } - - location /api { - proxy_pass http://nest-api-server/api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_cookie_domain nest-api-server juchum.info; - proxy_cookie_path / /; - } - - location /socket.io { - proxy_pass http://nest-api-server/socket.io; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } - } - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - #include /etc/nginx/conf.d/*.conf; -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts new file mode 100644 index 00000000..1d1fa103 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts @@ -0,0 +1,280 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Between, DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { + DetailDataQuery, + FinancialRatio, + isFinancialRatioData, + isProductDetail, + ProductDetail, + StockDetailQuery, +} from '../type/openapiDetailData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { getOpenApi } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { KospiStock } from '@/stock/domain/kospiStock.entity'; +import { Stock } from '@/stock/domain/stock.entity'; +import { StockDaily } from '@/stock/domain/stockData.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; + +@Injectable() +export class OpenapiDetailData { + private readonly financialUrl: string = + '/uapi/domestic-stock/v1/finance/financial-ratio'; + private readonly productUrl: string = + '/uapi/domestic-stock/v1/quotations/search-stock-info'; + private readonly intervals = 1000; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + //setTimeout(() => this.getDetailData(), 5000); + } + + @Cron('0 8 * * 1-5') + async getDetailData() { + if (process.env.NODE_ENV !== 'production') return; + const entityManager = this.datasource.manager; + const stocks = await entityManager.find(Stock); + const configCount = openApiToken.configs.length; + const chunkSize = Math.ceil(stocks.length / configCount); + + for (let i = 0; i < configCount; i++) { + this.logger.info(openApiToken.configs[i]); + const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); + this.getDetailDataChunk(chunk, openApiToken.configs[i]); + } + } + + private async saveDetailData(stockDetail: StockDetail) { + const manager = this.datasource.manager; + const entity = StockDetail; + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } + } + + private async saveKospiData(stockDetail: KospiStock) { + const manager = this.datasource.manager; + const entity = KospiStock; + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } + } + + private async calPer(eps: number): Promise { + if (eps <= 0) return NaN; + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const per = currentPrice / eps; + + if (isNaN(per)) return 0; + else return per; + } else { + return 0; + } + } + + private async calMarketCap(lstg: number) { + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const marketCap = lstg * currentPrice; + + if (isNaN(marketCap)) return 0; + else return marketCap; + } else { + return 0; + } + } + + private async get52WeeksLowHigh() { + const manager = this.datasource.manager; + const nowDate = new Date(); + const weeksAgoDate = this.getDate52WeeksAgo(); + // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 + const output = await manager.find(StockDaily, { + select: ['low', 'high'], + where: { + startTime: Between(weeksAgoDate, nowDate), + }, + }); + const result = output.reduce((prev, cur) => { + if (prev.low > cur.low) prev.low = cur.low; + if (prev.high < cur.high) prev.high = cur.high; + return cur; + }, new StockDaily()); + let low = 0; + let high = 0; + if (result.low && !isNaN(result.low)) low = result.low; + if (result.high && !isNaN(result.high)) high = result.high; + return { low, high }; + } + + private async makeStockDetailObject( + output1: FinancialRatio, + output2: ProductDetail, + stockId: string, + ): Promise { + const result = new StockDetail(); + result.stock = { id: stockId } as Stock; + result.marketCap = + (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; + result.eps = parseInt(output1.eps); + const { low, high } = await this.get52WeeksLowHigh(); + result.low52w = low; + result.high52w = high; + const eps = parseInt(output1.eps); + if (isNaN(eps)) result.eps = 0; + else result.eps = eps; + const per = await this.calPer(eps); + if (isNaN(per)) result.per = 0; + else result.per = per; + result.updatedAt = new Date(); + return result; + } + + private async makeKospiStockObject(output: ProductDetail, stockId: string) { + const ret = new KospiStock(); + ret.isKospi = output.kospi200_item_yn === 'Y' ? true : false; + ret.stock = { id: stockId } as Stock; + return ret; + } + + private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { + const dataQuery = this.getDetailDataQuery(stock.id!); + // 여기서 가져올 건 eps -> eps와 per 계산하자. + try { + const response = await getOpenApi( + this.financialUrl, + conf, + dataQuery, + TR_IDS.FINANCIAL_DATA, + ); + if (response.output) { + const output1 = response.output; + return output1[0]; + } + } catch (error) { + this.logger.error(error); + } + } + + private async getProductData(stock: Stock, conf: typeof openApiConfig) { + const defaultQuery = this.getFinancialDataQuery(stock.id!); + + // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 + try { + const response = await getOpenApi( + this.productUrl, + conf, + defaultQuery, + TR_IDS.PRODUCTION_DETAIL, + ); + if (response.output) { + const output2 = response.output; + return output2; + //return bufferToObject(output2); + } + } catch (error) { + this.logger.error(error); + } + } + + private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { + const output1 = await this.getFinancialRatio(stock, conf); + const output2 = await this.getProductData(stock, conf); + + this.logger.info(JSON.stringify(output1)); + this.logger.info(JSON.stringify(output2)); + if (isFinancialRatioData(output1) && isProductDetail(output2)) { + const stockDetail = await this.makeStockDetailObject( + output1, + output2, + stock.id!, + ); + this.saveDetailData(stockDetail); + const kospiStock = await this.makeKospiStockObject(output2, stock.id!); + this.saveKospiData(kospiStock); + + this.logger.info(`${stock.id!} is saved`); + } + } + + private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { + let delay = 0; + for await (const stock of chunk) { + setTimeout(() => this.getDetailDataDelay(stock, conf), delay); + delay += this.intervals; + } + } + + private getFinancialDataQuery( + stockId: string, + code: '300' | '301' | '302' | '306' = '300', + ): StockDetailQuery { + return { + pdno: stockId, + prdt_type_cd: code, + }; + } + + private getDetailDataQuery( + stockId: string, + divCode: 'J' = 'J', + classify: '0' | '1' = '0', + ): DetailDataQuery { + return { + fid_div_cls_code: classify, + fid_cond_mrkt_div_code: divCode, + fid_input_iscd: stockId, + }; + } + + private getDate52WeeksAgo(): Date { + const today = new Date(); + const weeksAgo = 52 * 7; + const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); + date52WeeksAgo.setHours(0, 0, 0, 0); + return date52WeeksAgo; + } +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts new file mode 100644 index 00000000..712d6d2d --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts @@ -0,0 +1,104 @@ +import { Inject } from '@nestjs/common'; +import { EntityManager } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { StockData, parseStockData } from '../type/openapiLiveData.type'; +import { decryptAES256 } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { KospiStock } from '@/stock/domain/kospiStock.entity'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; + +export class OpenapiLiveData { + public readonly TR_ID: string = 'H0STCNT0'; + private readonly WEBSOCKET_MAX: number = 40; + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly manager: EntityManager, + ) {} + + async getMessage(): Promise { + const kospi = await this.getKospiStockId(); + const config = openApiToken.configs; + const configLength = config.length; + const ret: string[] = []; + + for (let i = 0; i < configLength; i++) { + const stocks = kospi.splice( + i * this.WEBSOCKET_MAX, + (i + 1) * this.WEBSOCKET_MAX, + ); + for (const stock of stocks) { + ret.push(this.convertObjectToMessage(config[i], stock.id!)); + } + } + + return ret; + } + + private convertObjectToMessage( + config: typeof openApiConfig, + stockId: string, + ): string { + const message = { + header: { + approval_key: config.STOCK_WEBSOCKET_KEY!, + custtype: 'P', + tr_type: '1', + 'content-type': 'utf-8', + }, + body: { + input: { + tr_id: this.TR_ID, + tr_key: stockId, + }, + }, + }; + return JSON.stringify(message); + } + + private async getKospiStockId() { + const kospi = await this.manager.find(KospiStock, { + where: { + isKospi: true, + }, + }); + return kospi; + } + + private async saveLiveData(data: StockLiveData) { + await this.manager.save(StockLiveData, data); + } + + private convertLiveData(message: string[]): StockLiveData { + const stockData: StockData = parseStockData(message); + const stockLiveData = new StockLiveData(); + stockLiveData.currentPrice = parseFloat(stockData.STCK_PRPR); + stockLiveData.changeRate = parseFloat(stockData.PRDY_CTRT); + stockLiveData.volume = parseInt(stockData.CNTG_VOL); + stockLiveData.high = parseFloat(stockData.STCK_HGPR); + stockLiveData.low = parseFloat(stockData.STCK_LWPR); + stockLiveData.open = parseFloat(stockData.STCK_OPRC); + stockLiveData.previousClose = parseFloat(stockData.WGHN_AVRG_STCK_PRC); + stockLiveData.updatedAt = new Date(); + + return stockLiveData; + } + + public async output(message: Buffer, iv?: string, key?: string) { + const parsed = message.toString().split('|'); + if (parsed.length > 0) { + if (parsed[0] == '1' && iv && key) + parsed[3] = decryptAES256(parsed[3], iv, key); + if (parsed[1] !== this.TR_ID) return; + const stockData = parsed[3].split('^'); + const length = stockData.length / parseInt(parsed[2]); + const size = parseInt(parsed[2]); + const i = 0; + while (i < size) { + const data = stockData.splice(i * length, (i + 1) * length); + const liveData = this.convertLiveData(data); + this.saveLiveData(liveData); + } + } + } +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts new file mode 100644 index 00000000..003e7560 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts @@ -0,0 +1,156 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; + +import { + isMinuteData, + MinuteData, + UpdateStockQuery, +} from '../type/openapiMinuteData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { Stock } from '@/stock/domain/stock.entity'; +import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; + +const STOCK_CUT = 4; + +@Injectable() +export class OpenapiMinuteData { + private stock: Stock[][] = []; + private readonly entity = StockMinutely; + private readonly url: string = + '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; + private readonly intervals: number = 130; + private flip: number = 0; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + this.getStockData(); + } + + @Cron('0 1 * * 1-5') + async getStockData() { + if (process.env.NODE_ENV !== 'production') return; + const stock = await this.datasource.manager.findBy(Stock, { + isTrading: true, + }); + const stockSize = Math.ceil(stock.length / STOCK_CUT); + let i = 0; + this.stock = []; + while (i < STOCK_CUT) { + this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); + i++; + } + } + + private convertResToMinuteData( + stockId: string, + item: MinuteData, + time: string, + ) { + const stockPeriod = new StockData(); + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(item.stck_bsop_date.slice(0, 4)), + parseInt(item.stck_bsop_date.slice(4, 6)) - 1, + parseInt(item.stck_bsop_date.slice(6, 8)), + parseInt(time.slice(0, 2)), + parseInt(time.slice(2, 4)), + ); + stockPeriod.close = parseInt(item.stck_prpr); + stockPeriod.open = parseInt(item.stck_oprc); + stockPeriod.high = parseInt(item.stck_hgpr); + stockPeriod.low = parseInt(item.stck_lwpr); + stockPeriod.volume = parseInt(item.cntg_vol); + stockPeriod.createdAt = new Date(); + return stockPeriod; + } + + private isMarketOpenTime(time: string) { + const numberTime = parseInt(time); + return numberTime >= 90000 && numberTime <= 153000; + } + + private async saveMinuteData( + stockId: string, + item: MinuteData[], + time: string, + ) { + const manager = this.datasource.manager; + if (!this.isMarketOpenTime(time)) return; + const stockPeriod = item.map((val) => + this.convertResToMinuteData(stockId, val, time), + ); + manager.save(this.entity, stockPeriod); + } + + private async getMinuteDataInterval( + stockId: string, + time: string, + config: typeof openApiConfig, + ) { + const query = this.getUpdateStockQuery(stockId, time); + try { + const response = await getOpenApi( + this.url, + config, + query, + TR_IDS.MINUTE_DATA, + ); + let output; + if (response.output2) output = response.output2; + if (output && output[0] && isMinuteData(output[0])) { + this.saveMinuteData(stockId, output, time); + } + } catch (error) { + this.logger.warn(error); + } + } + + private async getMinuteDataChunk( + chunk: Stock[], + config: typeof openApiConfig, + ) { + const time = getCurrentTime(); + let interval = 0; + for await (const stock of chunk) { + setTimeout( + () => this.getMinuteDataInterval(stock.id!, time, config), + interval, + ); + interval += this.intervals; + } + } + + @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) + getMinuteData() { + if (process.env.NODE_ENV !== 'production') return; + const configCount = openApiToken.configs.length; + const stock = this.stock[this.flip % STOCK_CUT]; + this.flip++; + const chunkSize = Math.ceil(stock.length / configCount); + for (let i = 0; i < configCount; i++) { + const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); + this.getMinuteDataChunk(chunk, openApiToken.configs[i]); + } + } + + private getUpdateStockQuery( + stockId: string, + time: string, + isPastData: boolean = true, + marketCode: 'J' | 'W' = 'J', + ): UpdateStockQuery { + return { + fid_etc_cls_code: '', + fid_cond_mrkt_div_code: marketCode, + fid_input_iscd: stockId, + fid_input_hour_1: time, + fid_pw_data_incu_yn: isPastData ? 'Y' : 'N', + }; + } +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts new file mode 100644 index 00000000..f4268088 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts @@ -0,0 +1,218 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { DataSource, EntityManager } from 'typeorm'; +import { Logger } from 'winston'; +import { + ChartData, + isChartData, + ItemChartPriceQuery, + Period, +} from '../type/openapiPeriodData'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { + getOpenApi, + getPreviousDate, + getTodayDate, +} from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { Stock } from '@/stock/domain/stock.entity'; +import { + StockData, + StockDaily, + StockWeekly, + StockMonthly, + StockYearly, +} from '@/stock/domain/stockData.entity'; + +const DATE_TO_ENTITY = { + D: StockDaily, + W: StockWeekly, + M: StockMonthly, + Y: StockYearly, +}; + +const DATE_TO_MONTH = { + D: 3, + W: 6, + M: 12, + Y: 24, +}; + +const INTERVALS = 4000; + +@Injectable() +export class OpenapiPeriodData { + private readonly url: string = + '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + //this.getItemChartPriceCheck(); + } + + @Cron('0 1 * * 1-5') + async getItemChartPriceCheck() { + if (process.env.NODE_ENV !== 'production') return; + const stocks = await this.datasource.manager.find(Stock, { + where: { + isTrading: true, + }, + }); + const configCount = openApiToken.configs.length; + const chunkSize = Math.ceil(stocks.length / configCount); + + for (let i = 0; i < configCount; i++) { + const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); + this.getChartData(chunk, 'D'); + setTimeout(() => this.getChartData(chunk, 'W'), INTERVALS); + setTimeout(() => this.getChartData(chunk, 'M'), INTERVALS * 2); + setTimeout(() => this.getChartData(chunk, 'Y'), INTERVALS * 3); + } + } + + private async getChartData(chunk: Stock[], period: Period) { + const baseTime = INTERVALS * 4; + const entity = DATE_TO_ENTITY[period]; + + let time = 0; + for (const stock of chunk) { + time += baseTime; + setTimeout(() => this.processStockData(stock, period, entity), time); + } + } + + private async processStockData( + stock: Stock, + period: Period, + entity: typeof StockData, + ) { + const stockPeriod = new StockData(); + const manager = this.datasource.manager; + let configIdx = 0; + let end = getTodayDate(); + let start = getPreviousDate(end, 3); + let isFail = false; + + while (!isFail) { + configIdx = (configIdx + 1) % openApiToken.configs.length; + this.setStockPeriod(stockPeriod, stock.id!, end); + + // chart 데이터가 있는 지 확인 -> 리턴 + if (await this.existsChartData(stockPeriod, manager, entity)) return; + + const query = this.getItemChartPriceQuery(stock.id!, start, end, period); + + const output = await this.fetchChartData(query, configIdx); + + if (output) { + await this.saveChartData(entity, stock.id!, output); + ({ endDate: end, startDate: start } = this.updateDates(start, period)); + } else isFail = true; + } + } + + private setStockPeriod( + stockPeriod: StockData, + stockId: string, + endDate: string, + ): void { + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(endDate.slice(0, 4)), + parseInt(endDate.slice(4, 6)) - 1, + parseInt(endDate.slice(6, 8)), + ); + } + + private async fetchChartData(query: ItemChartPriceQuery, configIdx: number) { + try { + const response = await getOpenApi( + this.url, + openApiToken.configs[configIdx], + query, + TR_IDS.ITEM_CHART_PRICE, + ); + return response.output2 as ChartData[]; + } catch (error) { + this.logger.warn(error); + } + } + + private updateDates( + startDate: string, + period: Period, + ): { endDate: string; startDate: string } { + const endDate = getPreviousDate(startDate, DATE_TO_MONTH[period]); + startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]); + return { endDate, startDate }; + } + + private async existsChartData( + stock: StockData, + manager: EntityManager, + entity: typeof StockData, + ) { + return await manager.findOne(entity, { + where: { + stock: { id: stock.stock.id }, + createdAt: stock.startTime, + }, + }); + } + + private async insertChartData(stock: StockData, entity: typeof StockData) { + const manager = this.datasource.manager; + if (!(await this.existsChartData(stock, manager, entity))) { + await manager.save(entity, stock); + } + } + + private convertObjectToStockData(item: ChartData, stockId: string) { + const stockPeriod = new StockData(); + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(item.stck_bsop_date.slice(0, 4)), + parseInt(item.stck_bsop_date.slice(4, 6)) - 1, + parseInt(item.stck_bsop_date.slice(6, 8)), + ); + stockPeriod.close = parseInt(item.stck_clpr); + stockPeriod.open = parseInt(item.stck_oprc); + stockPeriod.high = parseInt(item.stck_hgpr); + stockPeriod.low = parseInt(item.stck_lwpr); + stockPeriod.volume = parseInt(item.acml_vol); + stockPeriod.createdAt = new Date(); + return stockPeriod; + } + + private async saveChartData( + entity: typeof StockData, + stockId: string, + data: ChartData[], + ) { + for (const item of data) { + if (!isChartData(item)) { + continue; + } + const stockPeriod = this.convertObjectToStockData(item, stockId); + await this.insertChartData(stockPeriod, entity); + } + } + + private getItemChartPriceQuery( + stockId: string, + startDate: string, + endDate: string, + period: Period, + marketCode: 'J' | 'W' = 'J', + ): ItemChartPriceQuery { + return { + fid_cond_mrkt_div_code: marketCode, + fid_input_iscd: stockId, + fid_input_date_1: startDate, + fid_input_date_2: endDate, + fid_period_div_code: period, + fid_org_adj_prc: 0, + }; + } +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts new file mode 100644 index 00000000..14ffd2e8 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts @@ -0,0 +1,114 @@ +import { Inject, UseFilters } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; +import { OpenapiException } from '../util/openapiCustom.error'; +import { postOpenApi } from '../util/openapiUtil.api'; +import { logger } from '@/configs/logger.config'; + +class OpenapiTokenApi { + private config: (typeof openApiConfig)[] = []; + constructor(@Inject('winston') private readonly logger: Logger) { + const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); + const api_keys = openApiConfig.STOCK_API_KEY!.split(','); + const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); + if ( + accounts.length === 0 || + accounts.length !== api_keys.length || + api_passwords.length !== api_keys.length + ) { + this.logger.warn('Open API Config Error'); + } + for (let i = 0; i < accounts.length; i++) { + this.config.push({ + STOCK_URL: openApiConfig.STOCK_URL, + STOCK_ACCOUNT: accounts[i], + STOCK_API_KEY: api_keys[i], + STOCK_API_PASSWORD: api_passwords[i], + }); + } + this.initAuthenValue(); + } + + get configs() { + //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 + return this.config; + } + + @UseFilters(OpenapiExceptionFilter) + private async initAuthenValue() { + const delay = 60000; + const delayMinute = delay / 1000 / 60; + + try { + await this.initAccessToken(); + await this.initWebSocketKey(); + } catch (error) { + if (error instanceof Error) { + this.logger.warn( + `Request failed: ${error.message}. Retrying in ${delayMinute} minute...`, + ); + } else { + this.logger.warn( + `Request failed. Retrying in ${delayMinute} minute...`, + ); + setTimeout(async () => { + await this.initAccessToken(); + await this.initWebSocketKey(); + }, delay); + } + } + } + + @Cron('50 0 * * 1-5') + private async initAccessToken() { + const updatedConfig = await Promise.all( + this.config.map(async (val) => { + val.STOCK_API_TOKEN = await this.getToken(val)!; + return val; + }), + ); + this.config = updatedConfig; + } + + @Cron('50 0 * * 1-5') + private async initWebSocketKey() { + const updatedConfig = await Promise.all( + this.config.map(async (val) => { + val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; + return val; + }), + ); + this.config = updatedConfig; + } + + private async getToken(config: typeof openApiConfig): Promise { + const body = { + grant_type: 'client_credentials', + appkey: config.STOCK_API_KEY, + appsecret: config.STOCK_API_PASSWORD, + }; + const tmp = await postOpenApi('/oauth2/tokenP', config, body); + if (!tmp.access_token) { + throw new OpenapiException('Access Token Failed', 403); + } + return tmp.access_token as string; + } + + private async getWebSocketKey(config: typeof openApiConfig): Promise { + const body = { + grant_type: 'client_credentials', + appkey: config.STOCK_API_KEY, + secretkey: config.STOCK_API_PASSWORD, + }; + const tmp = await postOpenApi('/oauth2/Approval', config, body); + if (!tmp.approval_key) { + throw new OpenapiException('WebSocket Key Failed', 403); + } + return tmp.approval_key as string; + } +} + +const openApiToken = new OpenapiTokenApi(logger); +export { openApiToken }; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts b/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts new file mode 100644 index 00000000..8aa12ea3 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts @@ -0,0 +1,17 @@ +import * as dotenv from 'dotenv'; + +dotenv.config(); + +export const openApiConfig: { + STOCK_URL: string | undefined; + STOCK_ACCOUNT: string | undefined; + STOCK_API_KEY: string | undefined; + STOCK_API_PASSWORD: string | undefined; + STOCK_API_TOKEN?: string; + STOCK_WEBSOCKET_KEY?: string; +} = { + STOCK_URL: process.env.STOCK_URL, + STOCK_ACCOUNT: process.env.STOCK_ACCOUNT, + STOCK_API_KEY: process.env.STOCK_API_KEY, + STOCK_API_PASSWORD: process.env.STOCK_API_PASSWORD, +}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts new file mode 100644 index 00000000..cb45c91c --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts @@ -0,0 +1,43 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; +import { OpenapiMinuteData } from './api/openapiMinuteData.api'; +import { OpenapiPeriodData } from './api/openapiPeriodData.api'; +import { OpenapiScraperService } from './openapi-scraper.service'; +import { WebsocketClient } from './websocketClient.service'; +import { Stock } from '@/stock/domain/stock.entity'; +import { + StockDaily, + StockMinutely, + StockMonthly, + StockWeekly, + StockYearly, +} from '@/stock/domain/stockData.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + Stock, + StockMinutely, + StockDaily, + StockWeekly, + StockMonthly, + StockYearly, + StockLiveData, + StockDetail, + ]), + ], + controllers: [], + providers: [ + OpenapiPeriodData, + OpenapiMinuteData, + OpenapiDetailData, + OpenapiScraperService, + OpenapiLiveData, + WebsocketClient, + ], +}) +export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts new file mode 100644 index 00000000..52c90179 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts @@ -0,0 +1,15 @@ +import { Injectable } from '@nestjs/common'; +import { DataSource } from 'typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiMinuteData } from './api/openapiMinuteData.api'; +import { OpenapiPeriodData } from './api/openapiPeriodData.api'; + +@Injectable() +export class OpenapiScraperService { + public constructor( + private datasource: DataSource, + private readonly openapiPeriodData: OpenapiPeriodData, + private readonly openapiMinuteData: OpenapiMinuteData, + private readonly openapiDetailData: OpenapiDetailData, + ) {} +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts new file mode 100644 index 00000000..38015d48 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts @@ -0,0 +1,214 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +/* eslint-disable max-lines-per-function */ + +export type DetailDataQuery = { + fid_cond_mrkt_div_code: 'J'; + fid_input_iscd: string; + fid_div_cls_code: '0' | '1'; +}; + +export type FinancialRatio = { + stac_yymm: string; // 결산 년월 + grs: string; // 매출액 증가율 + bsop_prfi_inrt: string; // 영업 이익 증가율 + ntin_inrt: string; // 순이익 증가율 + roe_val: string; // ROE 값 + eps: string; // EPS + sps: string; // 주당매출액 + bps: string; // BPS + rsrv_rate: string; // 유보 비율 + lblt_rate: string; // 부채 비율 +}; + +export function isFinancialRatioData(data: any): data is FinancialRatio { + return ( + data && + typeof data.stac_yymm === 'string' && + typeof data.grs === 'string' && + typeof data.bsop_prfi_inrt === 'string' && + typeof data.ntin_inrt === 'string' && + typeof data.roe_val === 'string' && + typeof data.eps === 'string' && + typeof data.sps === 'string' && + typeof data.bps === 'string' && + typeof data.rsrv_rate === 'string' && + typeof data.lblt_rate === 'string' + ); +} + +export type ProductDetail = { + pdno: string; // 상품번호 + prdt_type_cd: string; // 상품유형코드 + mket_id_cd: string; // 시장ID코드 + scty_grp_id_cd: string; // 증권그룹ID코드 + excg_dvsn_cd: string; // 거래소구분코드 + setl_mmdd: string; // 결산월일 + lstg_stqt: string; // 상장주수 - 이거 사용 + lstg_cptl_amt: string; // 상장자본금액 + cpta: string; // 자본금 + papr: string; // 액면가 + issu_pric: string; // 발행가격 + kospi200_item_yn: string; // 코스피200종목여부 - 이것도 사용 + scts_mket_lstg_dt: string; // 유가증권시장상장일자 + scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 + kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 + kosdaq_mket_lstg_abol_dt: string; // 코스닥시장상장폐지일자 + frbd_mket_lstg_dt: string; // 프리보드시장상장일자 + frbd_mket_lstg_abol_dt: string; // 프리보드시장상장폐지일자 + reits_kind_cd: string; // 리츠종류코드 + etf_dvsn_cd: string; // ETF구분코드 + oilf_fund_yn: string; // 유전펀드여부 + idx_bztp_lcls_cd: string; // 지수업종대분류코드 + idx_bztp_mcls_cd: string; // 지수업종중분류코드 + idx_bztp_scls_cd: string; // 지수업종소분류코드 + stck_kind_cd: string; // 주식종류코드 + mfnd_opng_dt: string; // 뮤추얼펀드개시일자 + mfnd_end_dt: string; // 뮤추얼펀드종료일자 + dpsi_erlm_cncl_dt: string; // 예탁등록취소일자 + etf_cu_qty: string; // ETFCU수량 + prdt_name: string; // 상품명 + prdt_name120: string; // 상품명120 + prdt_abrv_name: string; // 상품약어명 + std_pdno: string; // 표준상품번호 + prdt_eng_name: string; // 상품영문명 + prdt_eng_name120: string; // 상품영문명120 + prdt_eng_abrv_name: string; // 상품영문약어명 + dpsi_aptm_erlm_yn: string; // 예탁지정등록여부 + etf_txtn_type_cd: string; // ETF과세유형코드 + etf_type_cd: string; // ETF유형코드 + lstg_abol_dt: string; // 상장폐지일자 + nwst_odst_dvsn_cd: string; // 신주구주구분코드 + sbst_pric: string; // 대용가격 + thco_sbst_pric: string; // 당사대용가격 + thco_sbst_pric_chng_dt: string; // 당사대용가격변경일자 + tr_stop_yn: string; // 거래정지여부 + admn_item_yn: string; // 관리종목여부 + thdt_clpr: string; // 당일종가 + bfdy_clpr: string; // 전일종가 + clpr_chng_dt: string; // 종가변경일자 + std_idst_clsf_cd: string; // 표준산업분류코드 + std_idst_clsf_cd_name: string; // 표준산업분류코드명 + idx_bztp_lcls_cd_name: string; // 지수업종대분류코드명 + idx_bztp_mcls_cd_name: string; // 지수업종중분류코드명 + idx_bztp_scls_cd_name: string; // 지수업종소분류코드명 + ocr_no: string; // OCR번호 + crfd_item_yn: string; // 크라우드펀딩종목여부 + elec_scty_yn: string; // 전자증권여부 + issu_istt_cd: string; // 발행기관코드 + etf_chas_erng_rt_dbnb: string; // ETF추적수익율배수 + etf_etn_ivst_heed_item_yn: string; // ETFETN투자유의종목여부 + stln_int_rt_dvsn_cd: string; // 대주이자율구분코드 + frnr_psnl_lmt_rt: string; // 외국인개인한도비율 + lstg_rqsr_issu_istt_cd: string; // 상장신청인발행기관코드 + lstg_rqsr_item_cd: string; // 상장신청인종목코드 + trst_istt_issu_istt_cd: string; // 신탁기관발행기관코드 +}; + +export const isProductDetail = (data: any): data is ProductDetail => { + return ( + typeof data.pdno === 'string' && + typeof data.prdt_type_cd === 'string' && + typeof data.mket_id_cd === 'string' && + typeof data.scty_grp_id_cd === 'string' && + typeof data.excg_dvsn_cd === 'string' && + typeof data.setl_mmdd === 'string' && + typeof data.lstg_stqt === 'string' && + typeof data.lstg_cptl_amt === 'string' && + typeof data.cpta === 'string' && + typeof data.papr === 'string' && + typeof data.issu_pric === 'string' && + typeof data.kospi200_item_yn === 'string' && + typeof data.scts_mket_lstg_dt === 'string' && + typeof data.scts_mket_lstg_abol_dt === 'string' && + typeof data.kosdaq_mket_lstg_dt === 'string' && + typeof data.kosdaq_mket_lstg_abol_dt === 'string' && + typeof data.frbd_mket_lstg_dt === 'string' && + typeof data.frbd_mket_lstg_abol_dt === 'string' && + typeof data.reits_kind_cd === 'string' && + typeof data.etf_dvsn_cd === 'string' && + typeof data.oilf_fund_yn === 'string' && + typeof data.idx_bztp_lcls_cd === 'string' && + typeof data.idx_bztp_mcls_cd === 'string' && + typeof data.idx_bztp_scls_cd === 'string' && + typeof data.stck_kind_cd === 'string' && + typeof data.mfnd_opng_dt === 'string' && + typeof data.mfnd_end_dt === 'string' && + typeof data.dpsi_erlm_cncl_dt === 'string' && + typeof data.etf_cu_qty === 'string' && + typeof data.prdt_name === 'string' && + typeof data.prdt_name120 === 'string' && + typeof data.prdt_abrv_name === 'string' && + typeof data.std_pdno === 'string' && + typeof data.prdt_eng_name === 'string' && + typeof data.prdt_eng_name120 === 'string' && + typeof data.prdt_eng_abrv_name === 'string' && + typeof data.dpsi_aptm_erlm_yn === 'string' && + typeof data.etf_txtn_type_cd === 'string' && + typeof data.etf_type_cd === 'string' && + typeof data.lstg_abol_dt === 'string' && + typeof data.nwst_odst_dvsn_cd === 'string' && + typeof data.sbst_pric === 'string' && + typeof data.thco_sbst_pric === 'string' && + typeof data.thco_sbst_pric_chng_dt === 'string' && + typeof data.tr_stop_yn === 'string' && + typeof data.admn_item_yn === 'string' && + typeof data.thdt_clpr === 'string' && + typeof data.bfdy_clpr === 'string' && + typeof data.clpr_chng_dt === 'string' && + typeof data.std_idst_clsf_cd === 'string' && + typeof data.std_idst_clsf_cd_name === 'string' && + typeof data.idx_bztp_lcls_cd_name === 'string' && + typeof data.idx_bztp_mcls_cd_name === 'string' && + typeof data.idx_bztp_scls_cd_name === 'string' && + typeof data.ocr_no === 'string' && + typeof data.crfd_item_yn === 'string' && + typeof data.elec_scty_yn === 'string' && + typeof data.issu_istt_cd === 'string' && + typeof data.etf_chas_erng_rt_dbnb === 'string' && + typeof data.etf_etn_ivst_heed_item_yn === 'string' && + typeof data.stln_int_rt_dvsn_cd === 'string' && + typeof data.frnr_psnl_lmt_rt === 'string' && + typeof data.lstg_rqsr_issu_istt_cd === 'string' && + typeof data.lstg_rqsr_item_cd === 'string' && + typeof data.trst_istt_issu_istt_cd === 'string' + ); +}; + +export type StockDetailQuery = { + pdno: string; + prdt_type_cd: string; +}; + +//export type FinancialDetail = { +// stac_yymm: string; // 결산 년월 +// sale_account: string; // 매출액 +// sale_cost: string; // 매출원가 +// sale_totl_prfi: string; // 매출총이익 +// depr_cost: string; // 감가상각비 +// sell_mang: string; // 판매관리비 +// bsop_prti: string; // 영업이익 +// bsop_non_ernn: string; // 영업외수익 +// bsop_non_expn: string; // 영업외비용 +// op_prfi: string; // 영업이익 +// spec_prfi: string; // 특별이익 +// spec_loss: string; // 특별손실 +// thtr_ntin: string; // 세전순이익 +//}; + +//export const isFinancialDetail = (data: any): data is FinancialDetail => { +// return ( +// typeof data.stac_yymm === 'string' && +// typeof data.sale_account === 'string' && +// typeof data.sale_cost === 'string' && +// typeof data.sale_totl_prfi === 'string' && +// typeof data.depr_cost === 'string' && +// typeof data.sell_mang === 'string' && +// typeof data.bsop_prti === 'string' && +// typeof data.bsop_non_ernn === 'string' && +// typeof data.bsop_non_expn === 'string' && +// typeof data.op_prfi === 'string' && +// typeof data.spec_prfi === 'string' && +// typeof data.spec_loss === 'string' && +// typeof data.thtr_ntin === 'string' +// ); +//}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts new file mode 100644 index 00000000..d8041e7b --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts @@ -0,0 +1,152 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable max-lines-per-function */ + +export type StockData = { + MKSC_SHRN_ISCD: string; // 유가증권 단축 종목코드 + STCK_CNTG_HOUR: string; // 주식 체결 시간 + STCK_PRPR: string; // 주식 현재가 + PRDY_VRSS_SIGN: string; // 전일 대비 부호 + PRDY_VRSS: string; // 전일 대비 + PRDY_CTRT: string; // 전일 대비율 + WGHN_AVRG_STCK_PRC: string; // 가중 평균 주식 가격 + STCK_OPRC: string; // 주식 시가 + STCK_HGPR: string; // 주식 최고가 + STCK_LWPR: string; // 주식 최저가 + ASKP1: string; // 매도호가1 + BIDP1: string; // 매수호가1 + CNTG_VOL: string; // 체결 거래량 + ACML_VOL: string; // 누적 거래량 + ACML_TR_PBMN: string; // 누적 거래 대금 + SELN_CNTG_CSNU: string; // 매도 체결 건수 + SHNU_CNTG_CSNU: string; // 매수 체결 건수 + NTBY_CNTG_CSNU: string; // 순매수 체결 건수 + CTTR: string; // 체결강도 + SELN_CNTG_SMTN: string; // 총 매도 수량 + SHNU_CNTG_SMTN: string; // 총 매수 수량 + CCLD_DVSN: string; // 체결구분 + SHNU_RATE: string; // 매수비율 + PRDY_VOL_VRSS_ACML_VOL_RATE: string; // 전일 거래량 대비 등락율 + OPRC_HOUR: string; // 시가 시간 + OPRC_VRSS_PRPR_SIGN: string; // 시가대비구분 + OPRC_VRSS_PRPR: string; // 시가대비 + HGPR_HOUR: string; // 최고가 시간 + HGPR_VRSS_PRPR_SIGN: string; // 고가대비구분 + HGPR_VRSS_PRPR: string; // 고가대비 + LWPR_HOUR: string; // 최저가 시간 + LWPR_VRSS_PRPR_SIGN: string; // 저가대비구분 + LWPR_VRSS_PRPR: string; // 저가대비 + BSOP_DATE: string; // 영업 일자 + NEW_MKOP_CLS_CODE: string; // 신 장운영 구분 코드 + TRHT_YN: string; // 거래정지 여부 + ASKP_RSQN1: string; // 매도호가 잔량1 + BIDP_RSQN1: string; // 매수호가 잔량1 + TOTAL_ASKP_RSQN: string; // 총 매도호가 잔량 + TOTAL_BIDP_RSQN: string; // 총 매수호가 잔량 + VOL_TNRT: string; // 거래량 회전율 + PRDY_SMNS_HOUR_ACML_VOL: string; // 전일 동시간 누적 거래량 + PRDY_SMNS_HOUR_ACML_VOL_RATE: string; // 전일 동시간 누적 거래량 비율 + HOUR_CLS_CODE: string; // 시간 구분 코드 + MRKT_TRTM_CLS_CODE: string; // 임의종료구분코드 + VI_STND_PRC: string; // 정적VI발동기준가 +}; + +export function parseStockData(message: string[]): StockData { + return { + MKSC_SHRN_ISCD: message[0], + STCK_CNTG_HOUR: message[1], + STCK_PRPR: message[2], + PRDY_VRSS_SIGN: message[3], + PRDY_VRSS: message[4], + PRDY_CTRT: message[5], + WGHN_AVRG_STCK_PRC: message[6], + STCK_OPRC: message[7], + STCK_HGPR: message[8], + STCK_LWPR: message[9], + ASKP1: message[10], + BIDP1: message[11], + CNTG_VOL: message[12], + ACML_VOL: message[13], + ACML_TR_PBMN: message[14], + SELN_CNTG_CSNU: message[15], + SHNU_CNTG_CSNU: message[16], + NTBY_CNTG_CSNU: message[17], + CTTR: message[18], + SELN_CNTG_SMTN: message[19], + SHNU_CNTG_SMTN: message[20], + CCLD_DVSN: message[21], + SHNU_RATE: message[22], + PRDY_VOL_VRSS_ACML_VOL_RATE: message[23], + OPRC_HOUR: message[24], + OPRC_VRSS_PRPR_SIGN: message[25], + OPRC_VRSS_PRPR: message[26], + HGPR_HOUR: message[27], + HGPR_VRSS_PRPR_SIGN: message[28], + HGPR_VRSS_PRPR: message[29], + LWPR_HOUR: message[30], + LWPR_VRSS_PRPR_SIGN: message[31], + LWPR_VRSS_PRPR: message[32], + BSOP_DATE: message[33], + NEW_MKOP_CLS_CODE: message[34], + TRHT_YN: message[35], + ASKP_RSQN1: message[36], + BIDP_RSQN1: message[37], + TOTAL_ASKP_RSQN: message[38], + TOTAL_BIDP_RSQN: message[39], + VOL_TNRT: message[40], + PRDY_SMNS_HOUR_ACML_VOL: message[41], + PRDY_SMNS_HOUR_ACML_VOL_RATE: message[42], + HOUR_CLS_CODE: message[43], + MRKT_TRTM_CLS_CODE: message[44], + VI_STND_PRC: message[45], + }; +} + +export type OpenApiMessage = { + header: { + approval_key: string; + custtype: string; + tr_type: string; + 'content-type': string; + }; + body: { + input: { + tr_id: string; + tr_key: string; + }; + }; +}; + +export type MessageResponse = { + header: { + tr_id: string; + tr_key: string; + encrypt: string; + }; + body: { + rt_cd: string; + msg_cd: string; + msg1: string; + output?: { + iv: string; + key: string; + }; + }; +}; + +export function isMessageResponse(data: any): data is MessageResponse { + return ( + typeof data === 'object' && + data !== null && + typeof data.header === 'object' && + data.header !== null && + typeof data.header.tr_id === 'object' && + typeof data.header.tr_key === 'object' && + typeof data.header.encrypt === 'object' && + typeof data.body === 'object' && + data.body !== null && + typeof data.body.rt_cd === 'object' && + typeof data.body.msg_cd === 'object' && + typeof data.body.msg1 === 'object' && + typeof data.body.output === 'object' + ); +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts new file mode 100644 index 00000000..5deb2d9e --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts @@ -0,0 +1,33 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +export type MinuteData = { + stck_bsop_date: string; + stck_cntg_hour: string; + stck_prpr: string; + stck_oprc: string; + stck_hgpr: string; + stck_lwpr: string; + cntg_vol: string; + acml_tr_pbmn: string; +}; + +export type UpdateStockQuery = { + fid_etc_cls_code: string; + fid_cond_mrkt_div_code: 'J' | 'W'; + fid_input_iscd: string; + fid_input_hour_1: string; + fid_pw_data_incu_yn: 'Y' | 'N'; +}; + +export const isMinuteData = (data: any) => { + return ( + typeof data.stck_bsop_date === 'string' && + typeof data.stck_cntg_hour === 'string' && + typeof data.stck_prpr === 'string' && + typeof data.stck_oprc === 'string' && + typeof data.stck_hgpr === 'string' && + typeof data.stck_lwpr === 'string' && + typeof data.cntg_vol === 'string' && + typeof data.acml_tr_pbmn === 'string' + ); +}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts new file mode 100644 index 00000000..e4066f7c --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts @@ -0,0 +1,45 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +export type Period = 'D' | 'W' | 'M' | 'Y'; +export type ChartData = { + stck_bsop_date: string; + stck_clpr: string; + stck_oprc: string; + stck_hgpr: string; + stck_lwpr: string; + acml_vol: string; + acml_tr_pbmn: string; + flng_cls_code: string; + prtt_rate: string; + mod_yn: string; + prdy_vrss_sign: string; + prdy_vrss: string; + revl_issu_reas: string; +}; + +export type ItemChartPriceQuery = { + fid_cond_mrkt_div_code: 'J' | 'W'; + fid_input_iscd: string; + fid_input_date_1: string; + fid_input_date_2: string; + fid_period_div_code: Period; + fid_org_adj_prc: number; +}; + +export const isChartData = (data?: any) => { + return ( + data && + typeof data.stck_bsop_date === 'string' && + typeof data.stck_clpr === 'string' && + typeof data.stck_oprc === 'string' && + typeof data.stck_hgpr === 'string' && + typeof data.stck_lwpr === 'string' && + typeof data.acml_vol === 'string' && + typeof data.acml_tr_pbmn === 'string' && + typeof data.flng_cls_code === 'string' && + typeof data.prtt_rate === 'string' && + typeof data.mod_yn === 'string' && + typeof data.prdy_vrss_sign === 'string' && + typeof data.prdy_vrss === 'string' && + typeof data.revl_issu_reas === 'string' + ); +}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts new file mode 100644 index 00000000..6df0ca19 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts @@ -0,0 +1,13 @@ +export type TR_ID = + | 'FHKST03010100' + | 'FHKST03010200' + | 'FHKST66430300' + | 'HHKDB669107C0' + | 'CTPF1002R'; + +export const TR_IDS: Record = { + ITEM_CHART_PRICE: 'FHKST03010100', + MINUTE_DATA: 'FHKST03010200', + FINANCIAL_DATA: 'FHKST66430300', + PRODUCTION_DETAIL: 'CTPF1002R', +}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts new file mode 100644 index 00000000..1e0c3913 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts @@ -0,0 +1,13 @@ +import { HttpException, HttpStatus } from '@nestjs/common'; + +export class OpenapiException extends HttpException { + private error: unknown; + constructor(message: string, status: HttpStatus, error?: unknown) { + super(message, status); + this.error = error; + } + + public getError() { + return this.error; + } +} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts new file mode 100644 index 00000000..fa8f75b4 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts @@ -0,0 +1,115 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +import * as crypto from 'crypto'; +import { HttpStatus } from '@nestjs/common'; +import axios from 'axios'; +import { openApiConfig } from '../config/openapi.config'; +import { TR_ID } from '../type/openapiUtil.type'; +import { OpenapiException } from './openapiCustom.error'; + +const throwOpenapiException = (error: any) => { + if (error.message && error.response && error.response.status) { + throw new OpenapiException( + `Request failed: ${error.message}`, + error.response.status, + error, + ); + } else { + throw new OpenapiException( + `Unknown error: ${error.message || 'No message'}`, + HttpStatus.INTERNAL_SERVER_ERROR, + error, + ); + } +}; + +const postOpenApi = async ( + url: string, + config: typeof openApiConfig, + body: object, +) => { + try { + const response = await axios.post(config.STOCK_URL + url, body); + return response.data; + } catch (error) { + throwOpenapiException(error); + } +}; + +const getOpenApi = async ( + url: string, + config: typeof openApiConfig, + query: object, + tr_id: TR_ID, +) => { + try { + const response = await axios.get(config.STOCK_URL + url, { + params: query, + headers: { + Authorization: `Bearer ${config.STOCK_API_TOKEN}`, + appkey: config.STOCK_API_KEY, + appsecret: config.STOCK_API_PASSWORD, + tr_id, + custtype: 'P', + }, + }); + return response.data; + } catch (error) { + throwOpenapiException(error); + } +}; + +const getTodayDate = (): string => { + const today = new Date(); + return today.toISOString().split('T')[0].replace(/-/g, ''); +}; + +const getPreviousDate = (date: string, months: number): string => { + const currentDate = new Date( + date.slice(0, 4) + '-' + date.slice(4, 6) + '-' + date.slice(6, 8), + ); + currentDate.setMonth(currentDate.getMonth() - months); + return currentDate.toISOString().split('T')[0].replace(/-/g, ''); +}; + +const getCurrentTime = () => { + const now = new Date(); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + return `${hours}${minutes}${seconds}`; +}; + +const decryptAES256 = ( + encryptedText: string, + key: string, + iv: string, +): string => { + const decipher = crypto.createDecipheriv( + 'aes-256-cbc', + Buffer.from(key, 'hex'), + Buffer.from(iv, 'hex'), + ); + let decrypted = decipher.update(encryptedText, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + return decrypted; +}; + +const bufferToObject = (buffer: Buffer): any => { + try { + const jsonString = buffer.toString('utf-8'); + return JSON.parse(jsonString); + } catch (error) { + console.error('Failed to convert buffer to object:', error); + throw error; + } +}; + +export { + postOpenApi, + getOpenApi, + getTodayDate, + getPreviousDate, + getCurrentTime, + decryptAES256, + bufferToObject, +}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts new file mode 100644 index 00000000..554c3eff --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts @@ -0,0 +1,87 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Logger } from 'winston'; +import { WebSocket } from 'ws'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; + +@Injectable() +export class WebsocketClient { + private client: WebSocket; + private readonly reconnectInterval = 60000; + private readonly url = + process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; + + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly openapiLiveData: OpenapiLiveData, + ) { + if (process.env.NODE_ENV === 'production') { + this.connect(); + } + } + + // TODO : subscribe 구조로 리팩토링 + private subscribe() {} + + private message(data: any) { + this.logger.info(`Received message: ${data}`); + if (data.header && data.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(data)}`); + this.sendPong(); + return; + } + if (data.header && data.header.tr_id === 'H0STCNT0') { + return; + } + this.openapiLiveData.output(data); + } + + @Cron('0 2 * * 1-5') + private connect() { + this.client = new WebSocket(this.url); + + this.client.on('open', () => { + this.logger.info('WebSocket connection established'); + this.openapiLiveData.getMessage().then((val) => { + val.forEach((message) => this.sendMessage(message)); + }); + }); + + this.client.on('message', (data: any) => { + try { + this.message(data); + } catch (error) { + this.logger.info(error); + } + }); + + this.client.on('close', () => { + this.logger.warn( + `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, + ); + setTimeout(() => this.connect(), this.reconnectInterval); + }); + + this.client.on('error', (error: any) => { + this.logger.error(`WebSocket error: ${error.message}`); + }); + } + + private sendPong() { + const pongMessage = { + header: { tr_id: 'PINGPONG', datetime: new Date().toISOString() }, + }; + this.client.send(JSON.stringify(pongMessage)); + this.logger.info(`Sent PONG: ${JSON.stringify(pongMessage)}`); + } + + private sendMessage(message: string) { + if (this.client.readyState === WebSocket.OPEN) { + this.client.send(message); + this.logger.info(`Sent message: ${message}`); + } else { + this.logger.warn('WebSocket is not open. Message not sent.'); + } + } +} From 74875ebe2c24a66342ec545fca56048a332585bd Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 11:34:05 +0900 Subject: [PATCH 075/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openapiToken=20-?= =?UTF-8?q?=20custom=20filter=20=EC=82=AD=EC=A0=9C,=20=EB=B6=88=ED=95=84?= =?UTF-8?q?=EC=9A=94=ED=95=9C=20private=20=EC=82=AD=EC=A0=9C,=20try-catch?= =?UTF-8?q?=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../korea-stock-info/openapi/api/openapiDetailData.api.ts | 5 ++--- .../korea-stock-info/openapi/api/openapiToken.api.ts | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts index 1d1fa103..9be8e3ea 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts @@ -195,7 +195,7 @@ export class OpenapiDetailData { return output1[0]; } } catch (error) { - this.logger.error(error); + this.logger.warn(error); } } @@ -213,10 +213,9 @@ export class OpenapiDetailData { if (response.output) { const output2 = response.output; return output2; - //return bufferToObject(output2); } } catch (error) { - this.logger.error(error); + this.logger.warn(error); } } diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts index 14ffd2e8..6e6c88c5 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts @@ -1,8 +1,7 @@ -import { Inject, UseFilters } from '@nestjs/common'; +import { Inject } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; import { logger } from '@/configs/logger.config'; @@ -36,7 +35,6 @@ class OpenapiTokenApi { return this.config; } - @UseFilters(OpenapiExceptionFilter) private async initAuthenValue() { const delay = 60000; const delayMinute = delay / 1000 / 60; @@ -62,7 +60,7 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') - private async initAccessToken() { + async initAccessToken() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_API_TOKEN = await this.getToken(val)!; @@ -73,7 +71,7 @@ class OpenapiTokenApi { } @Cron('50 0 * * 1-5') - private async initWebSocketKey() { + async initWebSocketKey() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; From ff5e7bb6885830f2e55ed476100e7c79a705c6f3 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 16:15:50 +0900 Subject: [PATCH 076/112] =?UTF-8?q?=E2=9C=A8=20feat:=20priority=20queue=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/util/priorityQueue.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts new file mode 100644 index 00000000..a49e5822 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts @@ -0,0 +1,99 @@ +export class PriorityQueue { + private heap: { value: T; priority: number }[]; + + constructor() { + this.heap = []; + } + + private getParentIndex(index: number): number { + return Math.floor((index - 1) / 2); + } + + private getLeftChildIndex(index: number): number { + return index * 2 + 1; + } + + private getRightChildIndex(index: number): number { + return index * 2 + 2; + } + + private swap(i: number, j: number) { + [this.heap[i], this.heap[j]] = [this.heap[j], this.heap[i]]; + } + + private heapifyUp() { + let index = this.heap.length - 1; + while ( + index > 0 && + this.heap[index].priority < this.heap[this.getParentIndex(index)].priority + ) { + this.swap(index, this.getParentIndex(index)); + index = this.getParentIndex(index); + } + } + + private heapifyDown() { + let index = 0; + while (this.getLeftChildIndex(index) < this.heap.length) { + let smallerChildIndex = this.getLeftChildIndex(index); + const rightChildIndex = this.getRightChildIndex(index); + + if ( + rightChildIndex < this.heap.length && + this.heap[rightChildIndex].priority < + this.heap[smallerChildIndex].priority + ) { + smallerChildIndex = rightChildIndex; + } + + if (this.heap[index].priority <= this.heap[smallerChildIndex].priority) { + break; + } + + this.swap(index, smallerChildIndex); + index = smallerChildIndex; + } + } + + enqueue(value: T, priority: number) { + this.heap.push({ value, priority }); + this.heapifyUp(); + } + + dequeue(): T | undefined { + if (this.isEmpty()) { + return undefined; + } + + const root = this.heap[0]; + const last = this.heap.pop(); + + if (this.heap.length > 0 && last) { + this.heap[0] = last; + this.heapifyDown(); + } + + return root.value; + } + + peek(): T | undefined { + return this.heap.length > 0 ? this.heap[0].value : undefined; + } + + isEmpty(): boolean { + return this.heap.length === 0; + } +} + +const pq = new PriorityQueue(); + +pq.enqueue('Task A', 2); +pq.enqueue('Task B', 1); +pq.enqueue('Task C', 3); + +console.log(pq.dequeue()); // Task B +console.log(pq.peek()); // Task A +console.log(pq.dequeue()); // Task A +console.log(pq.isEmpty()); // false +console.log(pq.dequeue()); // Task C +console.log(pq.isEmpty()); // true From ab4e8219e7cc919fb487f03a4d6961ee9dba5e5b Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 16:16:40 +0900 Subject: [PATCH 077/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20type=20=EC=88=98?= =?UTF-8?q?=EC=A0=95=20-=20undefined=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20?= =?UTF-8?q?=ED=99=95=EC=9D=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/type/openapiLiveData.type.ts | 100 +++++++++--------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts index d8041e7b..e1687cee 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts @@ -50,57 +50,6 @@ export type StockData = { VI_STND_PRC: string; // 정적VI발동기준가 }; -export function parseStockData(message: string[]): StockData { - return { - MKSC_SHRN_ISCD: message[0], - STCK_CNTG_HOUR: message[1], - STCK_PRPR: message[2], - PRDY_VRSS_SIGN: message[3], - PRDY_VRSS: message[4], - PRDY_CTRT: message[5], - WGHN_AVRG_STCK_PRC: message[6], - STCK_OPRC: message[7], - STCK_HGPR: message[8], - STCK_LWPR: message[9], - ASKP1: message[10], - BIDP1: message[11], - CNTG_VOL: message[12], - ACML_VOL: message[13], - ACML_TR_PBMN: message[14], - SELN_CNTG_CSNU: message[15], - SHNU_CNTG_CSNU: message[16], - NTBY_CNTG_CSNU: message[17], - CTTR: message[18], - SELN_CNTG_SMTN: message[19], - SHNU_CNTG_SMTN: message[20], - CCLD_DVSN: message[21], - SHNU_RATE: message[22], - PRDY_VOL_VRSS_ACML_VOL_RATE: message[23], - OPRC_HOUR: message[24], - OPRC_VRSS_PRPR_SIGN: message[25], - OPRC_VRSS_PRPR: message[26], - HGPR_HOUR: message[27], - HGPR_VRSS_PRPR_SIGN: message[28], - HGPR_VRSS_PRPR: message[29], - LWPR_HOUR: message[30], - LWPR_VRSS_PRPR_SIGN: message[31], - LWPR_VRSS_PRPR: message[32], - BSOP_DATE: message[33], - NEW_MKOP_CLS_CODE: message[34], - TRHT_YN: message[35], - ASKP_RSQN1: message[36], - BIDP_RSQN1: message[37], - TOTAL_ASKP_RSQN: message[38], - TOTAL_BIDP_RSQN: message[39], - VOL_TNRT: message[40], - PRDY_SMNS_HOUR_ACML_VOL: message[41], - PRDY_SMNS_HOUR_ACML_VOL_RATE: message[42], - HOUR_CLS_CODE: message[43], - MRKT_TRTM_CLS_CODE: message[44], - VI_STND_PRC: message[45], - }; -} - export type OpenApiMessage = { header: { approval_key: string; @@ -150,3 +99,52 @@ export function isMessageResponse(data: any): data is MessageResponse { typeof data.body.output === 'object' ); } + +export const stockDataKeys = [ + 'MKSC_SHRN_ISCD', + 'STCK_CNTG_HOUR', + 'STCK_PRPR', + 'PRDY_VRSS_SIGN', + 'PRDY_VRSS', + 'PRDY_CTRT', + 'WGHN_AVRG_STCK_PRC', + 'STCK_OPRC', + 'STCK_HGPR', + 'STCK_LWPR', + 'ASKP1', + 'BIDP1', + 'CNTG_VOL', + 'ACML_VOL', + 'ACML_TR_PBMN', + 'SELN_CNTG_CSNU', + 'SHNU_CNTG_CSNU', + 'NTBY_CNTG_CSNU', + 'CTTR', + 'SELN_CNTG_SMTN', + 'SHNU_CNTG_SMTN', + 'CCLD_DVSN', + 'SHNU_RATE', + 'PRDY_VOL_VRSS_ACML_VOL_RATE', + 'OPRC_HOUR', + 'OPRC_VRSS_PRPR_SIGN', + 'OPRC_VRSS_PRPR', + 'HGPR_HOUR', + 'HGPR_VRSS_PRPR_SIGN', + 'HGPR_VRSS_PRPR', + 'LWPR_HOUR', + 'LWPR_VRSS_PRPR_SIGN', + 'LWPR_VRSS_PRPR', + 'BSOP_DATE', + 'NEW_MKOP_CLS_CODE', + 'TRHT_YN', + 'ASKP_RSQN1', + 'BIDP_RSQN1', + 'TOTAL_ASKP_RSQN', + 'TOTAL_BIDP_RSQN', + 'VOL_TNRT', + 'PRDY_SMNS_HOUR_ACML_VOL', + 'PRDY_SMNS_HOUR_ACML_VOL_RATE', + 'HOUR_CLS_CODE', + 'MRKT_TRTM_CLS_CODE', + 'VI_STND_PRC', +]; From b038411f98146f64929b1877e64888f5e7b3b6d3 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 16:17:31 +0900 Subject: [PATCH 078/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20websoc?= =?UTF-8?q?ket=20return=20=EA=B0=92=20parse=20=EB=B6=84=EB=A6=AC=ED=9B=84?= =?UTF-8?q?=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/parse/openapi.parser.spec.ts | 106 ++++++++++++++++++ .../openapi/parse/openapi.parser.ts | 38 +++++++ 2 files changed, 144 insertions(+) create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts create mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts new file mode 100644 index 00000000..37d3deb6 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts @@ -0,0 +1,106 @@ +/* eslint-disable max-lines-per-function */ +import { parseMessage } from './openapi.parser'; + +const answer = [ + { + STOCK_ID: '005930', + MKSC_SHRN_ISCD: 5930, + STCK_CNTG_HOUR: 93354, + STCK_PRPR: 71900, + PRDY_VRSS_SIGN: 5, + PRDY_VRSS: -100, + PRDY_CTRT: -0.14, + WGHN_AVRG_STCK_PRC: 72023.83, + STCK_OPRC: 72100, + STCK_HGPR: 72400, + STCK_LWPR: 71700, + ASKP1: 71900, + BIDP1: 71800, + CNTG_VOL: 1, + ACML_VOL: 3052507, + ACML_TR_PBMN: 219853241700, + SELN_CNTG_CSNU: 5105, + SHNU_CNTG_CSNU: 6937, + NTBY_CNTG_CSNU: 1832, + CTTR: 84.9, + SELN_CNTG_SMTN: 1366314, + SHNU_CNTG_SMTN: 1159996, + CCLD_DVSN: 1, + SHNU_RATE: 0.39, + PRDY_VOL_VRSS_ACML_VOL_RATE: 20.28, + OPRC_HOUR: 90020, + OPRC_VRSS_PRPR_SIGN: 5, + OPRC_VRSS_PRPR: -200, + HGPR_HOUR: 90820, + HGPR_VRSS_PRPR_SIGN: 5, + HGPR_VRSS_PRPR: -500, + LWPR_HOUR: 92619, + LWPR_VRSS_PRPR_SIGN: 2, + LWPR_VRSS_PRPR: 200, + BSOP_DATE: 20230612, + NEW_MKOP_CLS_CODE: 20, + TRHT_YN: 'N', + ASKP_RSQN1: 65945, + BIDP_RSQN1: 216924, + TOTAL_ASKP_RSQN: 1118750, + TOTAL_BIDP_RSQN: 2199206, + VOL_TNRT: 0.05, + PRDY_SMNS_HOUR_ACML_VOL: 2424142, + PRDY_SMNS_HOUR_ACML_VOL_RATE: 125.92, + HOUR_CLS_CODE: 0, + MRKT_TRTM_CLS_CODE: null, + VI_STND_PRC: 72100, + }, +]; + +describe('openapi parser test', () => { + test('parse json websocket data', () => { + const message = `{ + "header": { + "tr_id": "H0STCNT0", + "tr_key": "005930", + "encrypt": "N" + }, + "body": { + "rt_cd": "0", + "msg_cd": "OPSP0000", + "msg1": "SUBSCRIBE SUCCESS", + "output": { + "iv": "0123456789abcdef", + "key": "abcdefghijklmnopabcdefghijklmnop"} + } + }`; + + const result = parseMessage(message); + + expect(result).toEqual(JSON.parse(message)); + }); + + test('parse stockData', () => { + const message = + '0|H0STCNT0|001|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100'; + + const result = parseMessage(message); + + expect(result).toEqual(answer); + }); + + test('parse stockData', () => { + const message = + '0|H0STCNT0|002|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100^' + + '005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100'; + + const result = parseMessage(message); + + expect(result).toEqual([answer[0], answer[0]]); + }); +}); diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts new file mode 100644 index 00000000..fe3e7002 --- /dev/null +++ b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts @@ -0,0 +1,38 @@ +import { stockDataKeys } from '../type/openapiLiveData.type'; + +export const parseMessage = (data: string) => { + try { + return JSON.parse(data); + //eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return parseStockData(data); + } +}; +const FIELD_LENGTH: number = stockDataKeys.length; + +const parseStockData = (input: string) => { + const dataBlocks = input.split('|'); // 데이터 구분 + const results = []; + const size = parseInt(dataBlocks[2]); // 데이터 건수 + const rawData = dataBlocks[3]; + const values = rawData.split('^'); // 필드 구분자 '^' + + for (let i = 0; i < size; i++) { + //TODO : type narrowing require + const parsedData: Record = {}; + parsedData['STOCK_ID'] = values[i * FIELD_LENGTH]; + stockDataKeys.forEach((field: string, index: number) => { + const value = values[index + FIELD_LENGTH * i]; + if (!value) return (parsedData[field] = null); + + // 숫자형 필드 처리 + if (isNaN(parseInt(value))) { + parsedData[field] = value; // 문자열 그대로 저장 + } else { + parsedData[field] = parseFloat(value); // 숫자로 변환 + } + }); + results.push(parsedData); + } + return results; +}; From 0a56bb5bbb92b4aadf891da919c80c118dbc6585 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 16:18:31 +0900 Subject: [PATCH 079/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20parse?= =?UTF-8?q?=20stock=20data=EB=A5=BC=20=EB=8B=A4=EB=A5=B8=20=ED=95=A8?= =?UTF-8?q?=EC=88=98=EB=A1=9C=20=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiLiveData.api.ts | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts index 712d6d2d..10ce32fb 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts @@ -2,7 +2,11 @@ import { Inject } from '@nestjs/common'; import { EntityManager } from 'typeorm'; import { Logger } from 'winston'; import { openApiConfig } from '../config/openapi.config'; -import { StockData, parseStockData } from '../type/openapiLiveData.type'; +import { + StockData, + parseStockData, + stockDataKeys, +} from '../type/openapiLiveData.type'; import { decryptAES256 } from '../util/openapiUtil.api'; import { openApiToken } from './openapiToken.api'; import { KospiStock } from '@/stock/domain/kospiStock.entity'; @@ -11,6 +15,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; export class OpenapiLiveData { public readonly TR_ID: string = 'H0STCNT0'; private readonly WEBSOCKET_MAX: number = 40; + private readonly FIELD_LENGTH: number = stockDataKeys.length; constructor( @Inject('winston') private readonly logger: Logger, private readonly manager: EntityManager, @@ -84,15 +89,49 @@ export class OpenapiLiveData { return stockLiveData; } - public async output(message: Buffer, iv?: string, key?: string) { - const parsed = message.toString().split('|'); - if (parsed.length > 0) { - if (parsed[0] == '1' && iv && key) - parsed[3] = decryptAES256(parsed[3], iv, key); - if (parsed[1] !== this.TR_ID) return; - const stockData = parsed[3].split('^'); - const length = stockData.length / parseInt(parsed[2]); - const size = parseInt(parsed[2]); + private parseStockData = (input: string) => { + const dataBlocks = input.split('|'); // 데이터 구분 + const results = []; + const size = parseInt(dataBlocks[2]); // 데이터 건수 + const rawData = dataBlocks[3]; + const values = rawData.split('^'); // 필드 구분자 '^' + + for (let i = 0; i < size; i++) { + //TODO : type narrowing require + const parsedData: Record = {}; + parsedData['STOCK_ID'] = values[i * this.FIELD_LENGTH]; + stockDataKeys.forEach((field: string, index: number) => { + const value = values[index + this.FIELD_LENGTH * i]; + if (!value) return (parsedData[field] = null); + + // 숫자형 필드 처리 + if (isNaN(parseInt(value))) { + parsedData[field] = value; // 문자열 그대로 저장 + } else { + parsedData[field] = parseFloat(value); // 숫자로 변환 + } + }); + results.push(parsedData); + } + return results; + }; + + public async output( + message: Record | string, + iv?: string, + key?: string, + ) { + const data = + typeof message === 'string' + ? message.split('|') + : JSON.stringify(message); + if (typeof data !== 'string') { + if (data[0] == '1' && iv && key) + data[3] = decryptAES256(data[3], iv, key); + if (data[1] !== this.TR_ID) return; + const stockData = data[3].split('^'); + const length = stockData.length / parseInt(data[2]); + const size = parseInt(data[2]); const i = 0; while (i < size) { const data = stockData.splice(i * length, (i + 1) * length); From 41d2c85b63ed3fb09cd5c9681ef481ddff3b4100 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 19:31:42 +0900 Subject: [PATCH 080/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20web=20?= =?UTF-8?q?socket=20client=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/websocketClient.service.ts | 125 +++++++++++++----- 1 file changed, 94 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts index 554c3eff..e0e35f1f 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts @@ -4,13 +4,18 @@ import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; +import { parseMessage } from './parse/openapi.parser'; +import { openApiToken } from '@/scraper/openapi/api/openapiToken.api'; +import { openApiConfig } from '@/scraper/openapi/config/openapi.config'; +type TR_IDS = '0' | '1'; @Injectable() export class WebsocketClient { private client: WebSocket; private readonly reconnectInterval = 60000; private readonly url = process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; + private readonly clientStock: Set = new Set(); constructor( @Inject('winston') private readonly logger: Logger, @@ -22,50 +27,86 @@ export class WebsocketClient { } // TODO : subscribe 구조로 리팩토링 - private subscribe() {} + subscribe(stockId: string) { + this.clientStock.add(stockId); + // TODO : 하나의 config만 사용중. + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '1', + ); + this.sendMessage(message); + } - private message(data: any) { - this.logger.info(`Received message: ${data}`); - if (data.header && data.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.sendPong(); - return; - } - if (data.header && data.header.tr_id === 'H0STCNT0') { - return; - } - this.openapiLiveData.output(data); + discribe(stockId: string) { + this.clientStock.delete(stockId); + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '0', + ); + this.sendMessage(message); } - @Cron('0 2 * * 1-5') - private connect() { - this.client = new WebSocket(this.url); + private initDisconnect() { + this.client.on('close', () => { + this.logger.warn( + `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, + ); + }); + + this.client.on('error', (error: any) => { + this.logger.error(`WebSocket error: ${error.message}`); + setTimeout(() => this.connect(), this.reconnectInterval); + }); + } + private initOpen() { this.client.on('open', () => { this.logger.info('WebSocket connection established'); - this.openapiLiveData.getMessage().then((val) => { - val.forEach((message) => this.sendMessage(message)); - }); + for (const stockId of this.clientStock.keys()) { + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '1', + ); + this.sendMessage(message); + } }); + } - this.client.on('message', (data: any) => { + private initMessage() { + this.client.on('message', async (data) => { try { - this.message(data); + let message; + if (typeof data === 'object') { + message = data; + } else { + message = parseMessage(data as string); + } + if (message.header && message.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(data)}`); + this.sendPong(); + return; + } + if (message.header && message.header.tr_id === 'H0STCNT0') { + return; + } + this.logger.info(`Recived data : ${data}`); + const liveData = this.openapiLiveData.convertLiveData(message); + this.openapiLiveData.saveLiveData(liveData); } catch (error) { - this.logger.info(error); + this.logger.warn(error); } }); + } - this.client.on('close', () => { - this.logger.warn( - `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, - ); - setTimeout(() => this.connect(), this.reconnectInterval); - }); - - this.client.on('error', (error: any) => { - this.logger.error(`WebSocket error: ${error.message}`); - }); + @Cron('0 2 * * 1-5') + connect() { + this.client = new WebSocket(this.url); + this.initOpen(); + this.initMessage(); + this.initDisconnect(); } private sendPong() { @@ -76,6 +117,28 @@ export class WebsocketClient { this.logger.info(`Sent PONG: ${JSON.stringify(pongMessage)}`); } + private convertObjectToMessage( + config: typeof openApiConfig, + stockId: string, + tr_type: TR_IDS, + ): string { + const message = { + header: { + approval_key: config.STOCK_WEBSOCKET_KEY!, + custtype: 'P', + tr_type, + 'content-type': 'utf-8', + }, + body: { + input: { + tr_id: 'H0STCNT0', + tr_key: stockId, + }, + }, + }; + return JSON.stringify(message); + } + private sendMessage(message: string) { if (this.client.readyState === WebSocket.OPEN) { this.client.send(message); From 46aaf9bfba2b55cbba2ef7eef78ec8c8c001adcc Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 19:38:59 +0900 Subject: [PATCH 081/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20websoc?= =?UTF-8?q?ket=20Client=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81=20-=20initO?= =?UTF-8?q?pen,=20close=EB=93=B1=EC=9C=BC=EB=A1=9C=20=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/websocketClient.service.ts | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts index e0e35f1f..3073683e 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; -import { WebSocket } from 'ws'; +import { RawData, WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; import { parseMessage } from './parse/openapi.parser'; import { openApiToken } from '@/scraper/openapi/api/openapiToken.api'; @@ -78,18 +78,15 @@ export class WebsocketClient { private initMessage() { this.client.on('message', async (data) => { try { - let message; - if (typeof data === 'object') { - message = data; - } else { - message = parseMessage(data as string); - } - if (message.header && message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.sendPong(); - return; - } - if (message.header && message.header.tr_id === 'H0STCNT0') { + const message = this.parseMessage(data); + if (message.header) { + if (message.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(data)}`); + this.client.pong({ + tr_id: 'PINGPONG', + datetime: new Date().toISOString(), + }); + } return; } this.logger.info(`Recived data : ${data}`); @@ -101,6 +98,14 @@ export class WebsocketClient { }); } + private parseMessage(data: RawData) { + if (typeof data === 'object') { + return data; + } else { + return parseMessage(data as string); + } + } + @Cron('0 2 * * 1-5') connect() { this.client = new WebSocket(this.url); @@ -109,14 +114,6 @@ export class WebsocketClient { this.initDisconnect(); } - private sendPong() { - const pongMessage = { - header: { tr_id: 'PINGPONG', datetime: new Date().toISOString() }, - }; - this.client.send(JSON.stringify(pongMessage)); - this.logger.info(`Sent PONG: ${JSON.stringify(pongMessage)}`); - } - private convertObjectToMessage( config: typeof openApiConfig, stockId: string, From 8562e2095eccdf9b4bbea2c643b37101a3dc2431 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 21:08:24 +0900 Subject: [PATCH 082/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openapilive=20dat?= =?UTF-8?q?a=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiLiveData.api.ts | 155 ++-------- .../Decorator/openapiException.filter.ts | 33 -- .../openapi/api/openapiDetailData.api.ts | 282 ------------------ .../openapi/api/openapiLiveData.api.ts | 105 ------- .../openapi/api/openapiMinuteData.api.ts | 155 ---------- .../openapi/api/openapiPeriodData.api.ts | 215 ------------- .../scraper/openapi/api/openapiToken.api.ts | 114 ------- .../scraper/openapi/config/openapi.config.ts | 17 -- .../scraper/openapi/openapi-scraper.module.ts | 43 --- .../openapi/openapi-scraper.service.ts | 15 - .../openapi/parse/openapi.parser.spec.ts | 106 +++++++ .../scraper/openapi/parse/openapi.parser.ts | 38 +++ .../openapi/type/openapiDetailData.type.ts | 214 ------------- .../openapi/type/openapiLiveData.type.ts | 152 ---------- .../openapi/type/openapiMinuteData.type.ts | 33 -- .../scraper/openapi/type/openapiPeriodData.ts | 44 --- .../scraper/openapi/type/openapiUtil.type.ts | 13 - .../openapi/util/openapiCustom.error.ts | 13 - .../scraper/openapi/util/openapiUtil.api.ts | 115 ------- .../openapi/websocketClient.service.ts | 87 ------ 20 files changed, 168 insertions(+), 1781 deletions(-) delete mode 100644 packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts delete mode 100644 packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts delete mode 100644 packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts delete mode 100644 packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts delete mode 100644 packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts delete mode 100644 packages/backend/src/scraper/openapi/api/openapiToken.api.ts delete mode 100644 packages/backend/src/scraper/openapi/config/openapi.config.ts delete mode 100644 packages/backend/src/scraper/openapi/openapi-scraper.module.ts delete mode 100644 packages/backend/src/scraper/openapi/openapi-scraper.service.ts create mode 100644 packages/backend/src/scraper/openapi/parse/openapi.parser.spec.ts create mode 100644 packages/backend/src/scraper/openapi/parse/openapi.parser.ts delete mode 100644 packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts delete mode 100644 packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts delete mode 100644 packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts delete mode 100644 packages/backend/src/scraper/openapi/type/openapiPeriodData.ts delete mode 100644 packages/backend/src/scraper/openapi/type/openapiUtil.type.ts delete mode 100644 packages/backend/src/scraper/openapi/util/openapiCustom.error.ts delete mode 100644 packages/backend/src/scraper/openapi/util/openapiUtil.api.ts delete mode 100644 packages/backend/src/scraper/openapi/websocketClient.service.ts diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts index 10ce32fb..410ab4cd 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts @@ -1,143 +1,36 @@ -import { Inject } from '@nestjs/common'; -import { EntityManager } from 'typeorm'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { - StockData, - parseStockData, - stockDataKeys, -} from '../type/openapiLiveData.type'; -import { decryptAES256 } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { KospiStock } from '@/stock/domain/kospiStock.entity'; +import { DataSource } from 'typeorm'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; export class OpenapiLiveData { public readonly TR_ID: string = 'H0STCNT0'; - private readonly WEBSOCKET_MAX: number = 40; - private readonly FIELD_LENGTH: number = stockDataKeys.length; constructor( - @Inject('winston') private readonly logger: Logger, - private readonly manager: EntityManager, + private readonly datasource: DataSource, ) {} - async getMessage(): Promise { - const kospi = await this.getKospiStockId(); - const config = openApiToken.configs; - const configLength = config.length; - const ret: string[] = []; - - for (let i = 0; i < configLength; i++) { - const stocks = kospi.splice( - i * this.WEBSOCKET_MAX, - (i + 1) * this.WEBSOCKET_MAX, - ); - for (const stock of stocks) { - ret.push(this.convertObjectToMessage(config[i], stock.id!)); - } - } - - return ret; - } - - private convertObjectToMessage( - config: typeof openApiConfig, - stockId: string, - ): string { - const message = { - header: { - approval_key: config.STOCK_WEBSOCKET_KEY!, - custtype: 'P', - tr_type: '1', - 'content-type': 'utf-8', - }, - body: { - input: { - tr_id: this.TR_ID, - tr_key: stockId, - }, - }, - }; - return JSON.stringify(message); + async saveLiveData(data: StockLiveData[]) { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .insert() + .into(StockLiveData) + .values(data) + .execute(); } - private async getKospiStockId() { - const kospi = await this.manager.find(KospiStock, { - where: { - isKospi: true, - }, + convertLiveData(messages: Record[]) : StockLiveData[] { + const stockData: StockLiveData[] = []; + messages.map((message) => { + const stockLiveData = new StockLiveData(); + stockLiveData.currentPrice = parseFloat(message.STCK_PRPR); + stockLiveData.changeRate = parseFloat(message.PRDY_CTRT); + stockLiveData.volume = parseInt(message.CNTG_VOL); + stockLiveData.high = parseFloat(message.STCK_HGPR); + stockLiveData.low = parseFloat(message.STCK_LWPR); + stockLiveData.open = parseFloat(message.STCK_OPRC); + stockLiveData.previousClose = parseFloat(message.WGHN_AVRG_STCK_PRC); + stockLiveData.updatedAt = new Date(); + stockData.push(stockLiveData); }); - return kospi; - } - - private async saveLiveData(data: StockLiveData) { - await this.manager.save(StockLiveData, data); - } - - private convertLiveData(message: string[]): StockLiveData { - const stockData: StockData = parseStockData(message); - const stockLiveData = new StockLiveData(); - stockLiveData.currentPrice = parseFloat(stockData.STCK_PRPR); - stockLiveData.changeRate = parseFloat(stockData.PRDY_CTRT); - stockLiveData.volume = parseInt(stockData.CNTG_VOL); - stockLiveData.high = parseFloat(stockData.STCK_HGPR); - stockLiveData.low = parseFloat(stockData.STCK_LWPR); - stockLiveData.open = parseFloat(stockData.STCK_OPRC); - stockLiveData.previousClose = parseFloat(stockData.WGHN_AVRG_STCK_PRC); - stockLiveData.updatedAt = new Date(); - - return stockLiveData; - } - - private parseStockData = (input: string) => { - const dataBlocks = input.split('|'); // 데이터 구분 - const results = []; - const size = parseInt(dataBlocks[2]); // 데이터 건수 - const rawData = dataBlocks[3]; - const values = rawData.split('^'); // 필드 구분자 '^' - - for (let i = 0; i < size; i++) { - //TODO : type narrowing require - const parsedData: Record = {}; - parsedData['STOCK_ID'] = values[i * this.FIELD_LENGTH]; - stockDataKeys.forEach((field: string, index: number) => { - const value = values[index + this.FIELD_LENGTH * i]; - if (!value) return (parsedData[field] = null); - - // 숫자형 필드 처리 - if (isNaN(parseInt(value))) { - parsedData[field] = value; // 문자열 그대로 저장 - } else { - parsedData[field] = parseFloat(value); // 숫자로 변환 - } - }); - results.push(parsedData); - } - return results; - }; - - public async output( - message: Record | string, - iv?: string, - key?: string, - ) { - const data = - typeof message === 'string' - ? message.split('|') - : JSON.stringify(message); - if (typeof data !== 'string') { - if (data[0] == '1' && iv && key) - data[3] = decryptAES256(data[3], iv, key); - if (data[1] !== this.TR_ID) return; - const stockData = data[3].split('^'); - const length = stockData.length / parseInt(data[2]); - const size = parseInt(data[2]); - const i = 0; - while (i < size) { - const data = stockData.splice(i * length, (i + 1) * length); - const liveData = this.convertLiveData(data); - this.saveLiveData(liveData); - } - } + return stockData; } } diff --git a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts b/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts deleted file mode 100644 index a6c45ae9..00000000 --- a/packages/backend/src/scraper/openapi/Decorator/openapiException.filter.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - ExceptionFilter, - Catch, - HttpException, - HttpStatus, - Inject, -} from '@nestjs/common'; -import { Logger } from 'winston'; -import { OpenapiException } from '../util/openapiCustom.error'; - -@Catch() -export class OpenapiExceptionFilter implements ExceptionFilter { - constructor(@Inject('winston') private readonly logger: Logger) {} - - catch(exception: unknown) { - const status = - exception instanceof HttpException - ? exception.getStatus() - : HttpStatus.INTERNAL_SERVER_ERROR; - - const message = - exception instanceof HttpException - ? exception.getResponse() - : 'Internal server error'; - - const error = - exception instanceof OpenapiException ? exception.getError() : ''; - - this.logger.error( - `HTTP Status: ${status} Error Message: ${JSON.stringify(message)} Error : ${error}`, - ); - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts deleted file mode 100644 index 57b007a3..00000000 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { Inject, Injectable, UseFilters } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Between, DataSource } from 'typeorm'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; -import { - DetailDataQuery, - FinancialRatio, - isFinancialRatioData, - isProductDetail, - ProductDetail, - StockDetailQuery, -} from '../type/openapiDetailData.type'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { KospiStock } from '@/stock/domain/kospiStock.entity'; -import { Stock } from '@/stock/domain/stock.entity'; -import { StockDaily } from '@/stock/domain/stockData.entity'; -import { StockDetail } from '@/stock/domain/stockDetail.entity'; - -@Injectable() -export class OpenapiDetailData { - private readonly financialUrl: string = - '/uapi/domestic-stock/v1/finance/financial-ratio'; - private readonly productUrl: string = - '/uapi/domestic-stock/v1/quotations/search-stock-info'; - private readonly intervals = 1000; - constructor( - private readonly datasource: DataSource, - @Inject('winston') private readonly logger: Logger, - ) { - //setTimeout(() => this.getDetailData(), 5000); - } - - @Cron('0 8 * * 1-5') - @UseFilters(OpenapiExceptionFilter) - public async getDetailData() { - if (process.env.NODE_ENV !== 'production') return; - const entityManager = this.datasource.manager; - const stocks = await entityManager.find(Stock); - const configCount = openApiToken.configs.length; - const chunkSize = Math.ceil(stocks.length / configCount); - - for (let i = 0; i < configCount; i++) { - this.logger.info(openApiToken.configs[i]); - const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, openApiToken.configs[i]); - } - } - - private async saveDetailData(stockDetail: StockDetail) { - const manager = this.datasource.manager; - const entity = StockDetail; - const existingStockDetail = await manager.findOne(entity, { - where: { - stock: { id: stockDetail.stock.id }, - }, - }); - if (existingStockDetail) { - manager.update( - entity, - { stock: { id: stockDetail.stock.id } }, - stockDetail, - ); - } else { - manager.save(entity, stockDetail); - } - } - - private async saveKospiData(stockDetail: KospiStock) { - const manager = this.datasource.manager; - const entity = KospiStock; - const existingStockDetail = await manager.findOne(entity, { - where: { - stock: { id: stockDetail.stock.id }, - }, - }); - - if (existingStockDetail) { - manager.update( - entity, - { stock: { id: stockDetail.stock.id } }, - stockDetail, - ); - } else { - manager.save(entity, stockDetail); - } - } - - private async calPer(eps: number): Promise { - if (eps <= 0) return NaN; - const manager = this.datasource.manager; - const latestResult = await manager.find(StockDaily, { - skip: 0, - take: 1, - order: { createdAt: 'desc' }, - }); - // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 - if (latestResult && latestResult[0] && latestResult[0].close) { - const currentPrice = latestResult[0].close; - const per = currentPrice / eps; - - if (isNaN(per)) return 0; - else return per; - } else { - return 0; - } - } - - private async calMarketCap(lstg: number) { - const manager = this.datasource.manager; - const latestResult = await manager.find(StockDaily, { - skip: 0, - take: 1, - order: { createdAt: 'desc' }, - }); - - // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 - if (latestResult && latestResult[0] && latestResult[0].close) { - const currentPrice = latestResult[0].close; - const marketCap = lstg * currentPrice; - - if (isNaN(marketCap)) return 0; - else return marketCap; - } else { - return 0; - } - } - - private async get52WeeksLowHigh() { - const manager = this.datasource.manager; - const nowDate = new Date(); - const weeksAgoDate = this.getDate52WeeksAgo(); - // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 - const output = await manager.find(StockDaily, { - select: ['low', 'high'], - where: { - startTime: Between(weeksAgoDate, nowDate), - }, - }); - const result = output.reduce((prev, cur) => { - if (prev.low > cur.low) prev.low = cur.low; - if (prev.high < cur.high) prev.high = cur.high; - return cur; - }, new StockDaily()); - let low = 0; - let high = 0; - if (result.low && !isNaN(result.low)) low = result.low; - if (result.high && !isNaN(result.high)) high = result.high; - return { low, high }; - } - - private async makeStockDetailObject( - output1: FinancialRatio, - output2: ProductDetail, - stockId: string, - ): Promise { - const result = new StockDetail(); - result.stock = { id: stockId } as Stock; - result.marketCap = - (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; - result.eps = parseInt(output1.eps); - const { low, high } = await this.get52WeeksLowHigh(); - result.low52w = low; - result.high52w = high; - const eps = parseInt(output1.eps); - if (isNaN(eps)) result.eps = 0; - else result.eps = eps; - const per = await this.calPer(eps); - if (isNaN(per)) result.per = 0; - else result.per = per; - result.updatedAt = new Date(); - return result; - } - - private async makeKospiStockObject(output: ProductDetail, stockId: string) { - const ret = new KospiStock(); - ret.isKospi = output.kospi200_item_yn === 'Y' ? true : false; - ret.stock = { id: stockId } as Stock; - return ret; - } - - private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { - const dataQuery = this.getDetailDataQuery(stock.id!); - // 여기서 가져올 건 eps -> eps와 per 계산하자. - try { - const response = await getOpenApi( - this.financialUrl, - conf, - dataQuery, - TR_IDS.FINANCIAL_DATA, - ); - if (response.output) { - const output1 = response.output; - return output1[0]; - } - } catch (error) { - this.logger.error(error); - } - } - - private async getProductData(stock: Stock, conf: typeof openApiConfig) { - const defaultQuery = this.getFinancialDataQuery(stock.id!); - - // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 - try { - const response = await getOpenApi( - this.productUrl, - conf, - defaultQuery, - TR_IDS.PRODUCTION_DETAIL, - ); - if (response.output) { - const output2 = response.output; - return output2; - //return bufferToObject(output2); - } - } catch (error) { - this.logger.error(error); - } - } - - private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { - const output1 = await this.getFinancialRatio(stock, conf); - const output2 = await this.getProductData(stock, conf); - - this.logger.info(JSON.stringify(output1)); - this.logger.info(JSON.stringify(output2)); - if (isFinancialRatioData(output1) && isProductDetail(output2)) { - const stockDetail = await this.makeStockDetailObject( - output1, - output2, - stock.id!, - ); - this.saveDetailData(stockDetail); - const kospiStock = await this.makeKospiStockObject(output2, stock.id!); - this.saveKospiData(kospiStock); - - this.logger.info(`${stock.id!} is saved`); - } - } - - private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { - let delay = 0; - for await (const stock of chunk) { - setTimeout(() => this.getDetailDataDelay(stock, conf), delay); - delay += this.intervals; - } - } - - private getFinancialDataQuery( - stockId: string, - code: '300' | '301' | '302' | '306' = '300', - ): StockDetailQuery { - return { - pdno: stockId, - prdt_type_cd: code, - }; - } - - private getDetailDataQuery( - stockId: string, - divCode: 'J' = 'J', - classify: '0' | '1' = '0', - ): DetailDataQuery { - return { - fid_div_cls_code: classify, - fid_cond_mrkt_div_code: divCode, - fid_input_iscd: stockId, - }; - } - - private getDate52WeeksAgo(): Date { - const today = new Date(); - const weeksAgo = 52 * 7; - const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); - date52WeeksAgo.setHours(0, 0, 0, 0); - return date52WeeksAgo; - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts deleted file mode 100644 index 456cff90..00000000 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Inject } from '@nestjs/common'; -import { EntityManager } from 'typeorm'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { StockData, parseStockData } from '../type/openapiLiveData.type'; -import { decryptAES256 } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { KospiStock } from '@/stock/domain/kospiStock.entity'; -import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; - -export class OpenapiLiveData { - public readonly TR_ID: string = 'H0STCNT0'; - private readonly WEBSOCKET_MAX: number = 40; - constructor( - @Inject('winston') private readonly logger: Logger, - private readonly manager: EntityManager, - ) {} - - public async getMessage(): Promise { - const kospi = await this.getKospiStockId(); - const config = openApiToken.configs; - const configLength = config.length; - const ret: string[] = []; - - for (let i = 0; i < configLength; i++) { - const stocks = kospi.splice( - i * this.WEBSOCKET_MAX, - (i + 1) * this.WEBSOCKET_MAX, - ); - for (const stock of stocks) { - ret.push(this.convertObjectToMessage(config[i], stock.id!)); - } - } - - return ret; - } - - private convertObjectToMessage( - config: typeof openApiConfig, - stockId: string, - ): string { - const message = { - header: { - approval_key: config.STOCK_WEBSOCKET_KEY!, - custtype: 'P', - tr_type: '1', - 'content-type': 'utf-8', - }, - body: { - input: { - tr_id: this.TR_ID, - tr_key: stockId, - }, - }, - }; - return JSON.stringify(message); - } - - private async getKospiStockId() { - const kospi = await this.manager.find(KospiStock, { - where: { - isKospi: true, - }, - }); - return kospi; - } - - private async saveLiveData(data: StockLiveData) { - await this.manager.save(StockLiveData, data); - } - - private convertLiveData(message: string[]): StockLiveData { - const stockData: StockData = parseStockData(message); - const stockLiveData = new StockLiveData(); - stockLiveData.currentPrice = parseFloat(stockData.STCK_PRPR); - stockLiveData.changeRate = parseFloat(stockData.PRDY_CTRT); - stockLiveData.volume = parseInt(stockData.CNTG_VOL); - stockLiveData.high = parseFloat(stockData.STCK_HGPR); - stockLiveData.low = parseFloat(stockData.STCK_LWPR); - stockLiveData.open = parseFloat(stockData.STCK_OPRC); - stockLiveData.previousClose = parseFloat(stockData.WGHN_AVRG_STCK_PRC); - stockLiveData.updatedAt = new Date(); - - return stockLiveData; - } - - public async output(message: Buffer, iv?: string, key?: string) { - const parsed = message.toString().split('|'); - console.log(message.toString()); - if (parsed.length > 0) { - if (parsed[0] == '1' && iv && key) - parsed[3] = decryptAES256(parsed[3], iv, key); - if (parsed[1] !== this.TR_ID) return; - const stockData = parsed[3].split('^'); - const length = stockData.length / parseInt(parsed[2]); - const size = parseInt(parsed[2]); - const i = 0; - while (i < size) { - const data = stockData.splice(i * length, (i + 1) * length); - const liveData = this.convertLiveData(data); - this.saveLiveData(liveData); - } - } - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts deleted file mode 100644 index e050e3cd..00000000 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Injectable, UseFilters } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { DataSource } from 'typeorm'; -import { openApiConfig } from '../config/openapi.config'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; -import { - isMinuteData, - MinuteData, - UpdateStockQuery, -} from '../type/openapiMinuteData.type'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { Stock } from '@/stock/domain/stock.entity'; -import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; - -const STOCK_CUT = 4; - -@Injectable() -export class OpenapiMinuteData { - private stock: Stock[][] = []; - private readonly entity = StockMinutely; - private readonly url: string = - '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; - private readonly intervals: number = 130; - private flip: number = 0; - public constructor(private readonly datasource: DataSource) { - this.getStockData(); - } - - @Cron('0 1 * * 1-5') - @UseFilters(OpenapiExceptionFilter) - private async getStockData() { - if (process.env.NODE_ENV !== 'production') return; - const stock = await this.datasource.manager.findBy(Stock, { - isTrading: true, - }); - const stockSize = Math.ceil(stock.length / STOCK_CUT); - let i = 0; - this.stock = []; - while (i < STOCK_CUT) { - this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); - i++; - } - } - - private convertResToMinuteData( - stockId: string, - item: MinuteData, - time: string, - ) { - const stockPeriod = new StockData(); - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(item.stck_bsop_date.slice(0, 4)), - parseInt(item.stck_bsop_date.slice(4, 6)) - 1, - parseInt(item.stck_bsop_date.slice(6, 8)), - parseInt(time.slice(0, 2)), - parseInt(time.slice(2, 4)), - ); - stockPeriod.close = parseInt(item.stck_prpr); - stockPeriod.open = parseInt(item.stck_oprc); - stockPeriod.high = parseInt(item.stck_hgpr); - stockPeriod.low = parseInt(item.stck_lwpr); - stockPeriod.volume = parseInt(item.cntg_vol); - stockPeriod.createdAt = new Date(); - return stockPeriod; - } - - private isMarketOpenTime(time: string) { - const numberTime = parseInt(time); - return numberTime >= 90000 && numberTime <= 153000; - } - - private async saveMinuteData( - stockId: string, - item: MinuteData[], - time: string, - ) { - const manager = this.datasource.manager; - if (!this.isMarketOpenTime(time)) return; - const stockPeriod = item.map((val) => - this.convertResToMinuteData(stockId, val, time), - ); - manager.save(this.entity, stockPeriod); - } - - private async getMinuteDataInterval( - stockId: string, - time: string, - config: typeof openApiConfig, - ) { - const query = this.getUpdateStockQuery(stockId, time); - try { - const response = await getOpenApi( - this.url, - config, - query, - TR_IDS.MINUTE_DATA, - ); - let output; - if (response.output2) output = response.output2; - if (output && output[0] && isMinuteData(output[0])) { - this.saveMinuteData(stockId, output, time); - } - } catch (error) { - console.error(error); - } - } - - @UseFilters(OpenapiExceptionFilter) - private async getMinuteDataChunk( - chunk: Stock[], - config: typeof openApiConfig, - ) { - const time = getCurrentTime(); - let interval = 0; - for await (const stock of chunk) { - setTimeout( - () => this.getMinuteDataInterval(stock.id!, time, config), - interval, - ); - interval += this.intervals; - } - } - - @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) - @UseFilters(OpenapiExceptionFilter) - private getMinuteData() { - if (process.env.NODE_ENV !== 'production') return; - const configCount = openApiToken.configs.length; - const stock = this.stock[this.flip % STOCK_CUT]; - this.flip++; - const chunkSize = Math.ceil(stock.length / configCount); - for (let i = 0; i < configCount; i++) { - const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); - this.getMinuteDataChunk(chunk, openApiToken.configs[i]); - } - } - - private getUpdateStockQuery( - stockId: string, - time: string, - isPastData: boolean = true, - marketCode: 'J' | 'W' = 'J', - ): UpdateStockQuery { - return { - fid_etc_cls_code: '', - fid_cond_mrkt_div_code: marketCode, - fid_input_iscd: stockId, - fid_input_hour_1: time, - fid_pw_data_incu_yn: isPastData ? 'Y' : 'N', - }; - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts deleted file mode 100644 index 64db1d9b..00000000 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { Inject, Injectable, UseFilters } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { DataSource, EntityManager } from 'typeorm'; -import { Logger } from 'winston'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; -import { - ChartData, - isChartData, - ItemChartPriceQuery, - Period, -} from '../type/openapiPeriodData'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { - getOpenApi, - getPreviousDate, - getTodayDate, -} from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { Stock } from '@/stock/domain/stock.entity'; -import { - StockData, - StockDaily, - StockWeekly, - StockMonthly, - StockYearly, -} from '@/stock/domain/stockData.entity'; - -const DATE_TO_ENTITY = { - D: StockDaily, - W: StockWeekly, - M: StockMonthly, - Y: StockYearly, -}; - -const DATE_TO_MONTH = { - D: 3, - W: 6, - M: 12, - Y: 24, -}; - -const INTERVALS = 4000; - -@Injectable() -export class OpenapiPeriodData { - private readonly url: string = - '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; - public constructor( - private readonly datasource: DataSource, - @Inject('winston') private readonly logger: Logger, - ) { - //this.getItemChartPriceCheck(); - } - - @Cron('0 1 * * 1-5') - @UseFilters(OpenapiExceptionFilter) - public async getItemChartPriceCheck() { - if (process.env.NODE_ENV !== 'production') return; - const entityManager = this.datasource.manager; - const stocks = await entityManager.find(Stock); - const configCount = openApiToken.configs.length; - const chunkSize = Math.ceil(stocks.length / configCount); - - for (let i = 0; i < configCount; i++) { - const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getChartData(chunk, 'D'); - setTimeout(() => this.getChartData(chunk, 'W'), INTERVALS); - setTimeout(() => this.getChartData(chunk, 'M'), INTERVALS * 2); - setTimeout(() => this.getChartData(chunk, 'Y'), INTERVALS * 3); - } - } - - private async getChartData(chunk: Stock[], period: Period) { - const baseTime = INTERVALS * 4; - const entity = DATE_TO_ENTITY[period]; - const manager = this.datasource.manager; - - let time = 0; - for (const stock of chunk) { - time += baseTime; - setTimeout( - () => this.processStockData(stock, period, entity, manager), - time, - ); - } - } - - private async processStockData( - stock: Stock, - period: Period, - entity: typeof StockData, - manager: EntityManager, - ) { - const stockPeriod = new StockData(); - let configIdx = 0; - let end = getTodayDate(); - let start = getPreviousDate(end, 3); - let isFail = false; - - while (isFail) { - configIdx = (configIdx + 1) % openApiToken.configs.length; - this.setStockPeriod(stockPeriod, stock.id!, end); - - if (await this.existsChartData(stockPeriod, manager, entity)) return; - - const query = this.getItemChartPriceQuery(stock.id!, start, end, period); - - const output = await this.fetchChartData(query, configIdx); - - if (output && isChartData(output[0])) { - await this.saveChartData(entity, stock.id!, output); - ({ endDate: end, startDate: start } = this.updateDates(start, period)); - } else isFail = true; - } - } - - private setStockPeriod( - stockPeriod: StockData, - stockId: string, - endDate: string, - ): void { - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(endDate.slice(0, 4)), - parseInt(endDate.slice(4, 6)) - 1, - parseInt(endDate.slice(6, 8)), - ); - } - - private async fetchChartData(query: ItemChartPriceQuery, configIdx: number) { - try { - const response = await getOpenApi( - this.url, - openApiToken.configs[configIdx], - query, - TR_IDS.ITEM_CHART_PRICE, - ); - return response.output2 as ChartData[]; - } catch (error) { - this.logger.error(error); - } - } - - private updateDates( - startDate: string, - period: Period, - ): { endDate: string; startDate: string } { - const endDate = getPreviousDate(startDate, DATE_TO_MONTH[period]); - startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]); - return { endDate, startDate }; - } - - private async existsChartData( - stock: StockData, - manager: EntityManager, - entity: typeof StockData, - ) { - return await manager.findOne(entity, { - where: { - stock: { id: stock.stock.id }, - createdAt: stock.startTime, - }, - }); - } - - private async insertChartData(stock: StockData, entity: typeof StockData) { - const manager = this.datasource.manager; - if (!(await this.existsChartData(stock, manager, entity))) { - await manager.save(entity, stock); - } - } - - private async saveChartData( - entity: typeof StockData, - stockId: string, - data: ChartData[], - ) { - for (const item of data) { - if (!item || !item.stck_bsop_date) { - continue; - } - const stockPeriod = new StockData(); - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(item.stck_bsop_date.slice(0, 4)), - parseInt(item.stck_bsop_date.slice(4, 6)) - 1, - parseInt(item.stck_bsop_date.slice(6, 8)), - ); - stockPeriod.close = parseInt(item.stck_clpr); - stockPeriod.open = parseInt(item.stck_oprc); - stockPeriod.high = parseInt(item.stck_hgpr); - stockPeriod.low = parseInt(item.stck_lwpr); - stockPeriod.volume = parseInt(item.acml_vol); - stockPeriod.createdAt = new Date(); - await this.insertChartData(stockPeriod, entity); - } - } - - private getItemChartPriceQuery( - stockId: string, - startDate: string, - endDate: string, - period: Period, - marketCode: 'J' | 'W' = 'J', - ): ItemChartPriceQuery { - return { - fid_cond_mrkt_div_code: marketCode, - fid_input_iscd: stockId, - fid_input_date_1: startDate, - fid_input_date_2: endDate, - fid_period_div_code: period, - fid_org_adj_prc: 0, - }; - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts deleted file mode 100644 index fa4901a8..00000000 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { Inject, UseFilters } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { OpenapiExceptionFilter } from '../Decorator/openapiException.filter'; -import { OpenapiException } from '../util/openapiCustom.error'; -import { postOpenApi } from '../util/openapiUtil.api'; -import { logger } from '@/configs/logger.config'; - -class OpenapiTokenApi { - private config: (typeof openApiConfig)[] = []; - public constructor(@Inject('winston') private readonly logger: Logger) { - const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); - const api_keys = openApiConfig.STOCK_API_KEY!.split(','); - const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); - if ( - accounts.length === 0 || - accounts.length !== api_keys.length || - api_passwords.length !== api_keys.length - ) { - this.logger.warn('Open API Config Error'); - } - for (let i = 0; i < accounts.length; i++) { - this.config.push({ - STOCK_URL: openApiConfig.STOCK_URL, - STOCK_ACCOUNT: accounts[i], - STOCK_API_KEY: api_keys[i], - STOCK_API_PASSWORD: api_passwords[i], - }); - } - this.initAuthenValue(); - } - - public get configs() { - //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 - return this.config; - } - - @UseFilters(OpenapiExceptionFilter) - private async initAuthenValue() { - const delay = 60000; - const delayMinute = delay / 1000 / 60; - - try { - await this.initAccessToken(); - await this.initWebSocketKey(); - } catch (error) { - if (error instanceof Error) { - this.logger.warn( - `Request failed: ${error.message}. Retrying in ${delayMinute} minute...`, - ); - } else { - this.logger.warn( - `Request failed. Retrying in ${delayMinute} minute...`, - ); - setTimeout(async () => { - await this.initAccessToken(); - await this.initWebSocketKey(); - }, delay); - } - } - } - - @Cron('50 0 * * 1-5') - private async initAccessToken() { - const updatedConfig = await Promise.all( - this.config.map(async (val) => { - val.STOCK_API_TOKEN = await this.getToken(val)!; - return val; - }), - ); - this.config = updatedConfig; - } - - @Cron('50 0 * * 1-5') - private async initWebSocketKey() { - const updatedConfig = await Promise.all( - this.config.map(async (val) => { - val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; - return val; - }), - ); - this.config = updatedConfig; - } - - private async getToken(config: typeof openApiConfig): Promise { - const body = { - grant_type: 'client_credentials', - appkey: config.STOCK_API_KEY, - appsecret: config.STOCK_API_PASSWORD, - }; - const tmp = await postOpenApi('/oauth2/tokenP', config, body); - if (!tmp.access_token) { - throw new OpenapiException('Access Token Failed', 403); - } - return tmp.access_token as string; - } - - private async getWebSocketKey(config: typeof openApiConfig): Promise { - const body = { - grant_type: 'client_credentials', - appkey: config.STOCK_API_KEY, - secretkey: config.STOCK_API_PASSWORD, - }; - const tmp = await postOpenApi('/oauth2/Approval', config, body); - if (!tmp.approval_key) { - throw new OpenapiException('WebSocket Key Failed', 403); - } - return tmp.approval_key as string; - } -} - -const openApiToken = new OpenapiTokenApi(logger); -export { openApiToken }; diff --git a/packages/backend/src/scraper/openapi/config/openapi.config.ts b/packages/backend/src/scraper/openapi/config/openapi.config.ts deleted file mode 100644 index 8aa12ea3..00000000 --- a/packages/backend/src/scraper/openapi/config/openapi.config.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as dotenv from 'dotenv'; - -dotenv.config(); - -export const openApiConfig: { - STOCK_URL: string | undefined; - STOCK_ACCOUNT: string | undefined; - STOCK_API_KEY: string | undefined; - STOCK_API_PASSWORD: string | undefined; - STOCK_API_TOKEN?: string; - STOCK_WEBSOCKET_KEY?: string; -} = { - STOCK_URL: process.env.STOCK_URL, - STOCK_ACCOUNT: process.env.STOCK_ACCOUNT, - STOCK_API_KEY: process.env.STOCK_API_KEY, - STOCK_API_PASSWORD: process.env.STOCK_API_PASSWORD, -}; diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts deleted file mode 100644 index cb45c91c..00000000 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { OpenapiDetailData } from './api/openapiDetailData.api'; -import { OpenapiLiveData } from './api/openapiLiveData.api'; -import { OpenapiMinuteData } from './api/openapiMinuteData.api'; -import { OpenapiPeriodData } from './api/openapiPeriodData.api'; -import { OpenapiScraperService } from './openapi-scraper.service'; -import { WebsocketClient } from './websocketClient.service'; -import { Stock } from '@/stock/domain/stock.entity'; -import { - StockDaily, - StockMinutely, - StockMonthly, - StockWeekly, - StockYearly, -} from '@/stock/domain/stockData.entity'; -import { StockDetail } from '@/stock/domain/stockDetail.entity'; -import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; - -@Module({ - imports: [ - TypeOrmModule.forFeature([ - Stock, - StockMinutely, - StockDaily, - StockWeekly, - StockMonthly, - StockYearly, - StockLiveData, - StockDetail, - ]), - ], - controllers: [], - providers: [ - OpenapiPeriodData, - OpenapiMinuteData, - OpenapiDetailData, - OpenapiScraperService, - OpenapiLiveData, - WebsocketClient, - ], -}) -export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts deleted file mode 100644 index 52c90179..00000000 --- a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; -import { OpenapiDetailData } from './api/openapiDetailData.api'; -import { OpenapiMinuteData } from './api/openapiMinuteData.api'; -import { OpenapiPeriodData } from './api/openapiPeriodData.api'; - -@Injectable() -export class OpenapiScraperService { - public constructor( - private datasource: DataSource, - private readonly openapiPeriodData: OpenapiPeriodData, - private readonly openapiMinuteData: OpenapiMinuteData, - private readonly openapiDetailData: OpenapiDetailData, - ) {} -} diff --git a/packages/backend/src/scraper/openapi/parse/openapi.parser.spec.ts b/packages/backend/src/scraper/openapi/parse/openapi.parser.spec.ts new file mode 100644 index 00000000..37d3deb6 --- /dev/null +++ b/packages/backend/src/scraper/openapi/parse/openapi.parser.spec.ts @@ -0,0 +1,106 @@ +/* eslint-disable max-lines-per-function */ +import { parseMessage } from './openapi.parser'; + +const answer = [ + { + STOCK_ID: '005930', + MKSC_SHRN_ISCD: 5930, + STCK_CNTG_HOUR: 93354, + STCK_PRPR: 71900, + PRDY_VRSS_SIGN: 5, + PRDY_VRSS: -100, + PRDY_CTRT: -0.14, + WGHN_AVRG_STCK_PRC: 72023.83, + STCK_OPRC: 72100, + STCK_HGPR: 72400, + STCK_LWPR: 71700, + ASKP1: 71900, + BIDP1: 71800, + CNTG_VOL: 1, + ACML_VOL: 3052507, + ACML_TR_PBMN: 219853241700, + SELN_CNTG_CSNU: 5105, + SHNU_CNTG_CSNU: 6937, + NTBY_CNTG_CSNU: 1832, + CTTR: 84.9, + SELN_CNTG_SMTN: 1366314, + SHNU_CNTG_SMTN: 1159996, + CCLD_DVSN: 1, + SHNU_RATE: 0.39, + PRDY_VOL_VRSS_ACML_VOL_RATE: 20.28, + OPRC_HOUR: 90020, + OPRC_VRSS_PRPR_SIGN: 5, + OPRC_VRSS_PRPR: -200, + HGPR_HOUR: 90820, + HGPR_VRSS_PRPR_SIGN: 5, + HGPR_VRSS_PRPR: -500, + LWPR_HOUR: 92619, + LWPR_VRSS_PRPR_SIGN: 2, + LWPR_VRSS_PRPR: 200, + BSOP_DATE: 20230612, + NEW_MKOP_CLS_CODE: 20, + TRHT_YN: 'N', + ASKP_RSQN1: 65945, + BIDP_RSQN1: 216924, + TOTAL_ASKP_RSQN: 1118750, + TOTAL_BIDP_RSQN: 2199206, + VOL_TNRT: 0.05, + PRDY_SMNS_HOUR_ACML_VOL: 2424142, + PRDY_SMNS_HOUR_ACML_VOL_RATE: 125.92, + HOUR_CLS_CODE: 0, + MRKT_TRTM_CLS_CODE: null, + VI_STND_PRC: 72100, + }, +]; + +describe('openapi parser test', () => { + test('parse json websocket data', () => { + const message = `{ + "header": { + "tr_id": "H0STCNT0", + "tr_key": "005930", + "encrypt": "N" + }, + "body": { + "rt_cd": "0", + "msg_cd": "OPSP0000", + "msg1": "SUBSCRIBE SUCCESS", + "output": { + "iv": "0123456789abcdef", + "key": "abcdefghijklmnopabcdefghijklmnop"} + } + }`; + + const result = parseMessage(message); + + expect(result).toEqual(JSON.parse(message)); + }); + + test('parse stockData', () => { + const message = + '0|H0STCNT0|001|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100'; + + const result = parseMessage(message); + + expect(result).toEqual(answer); + }); + + test('parse stockData', () => { + const message = + '0|H0STCNT0|002|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100^' + + '005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + + '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + + '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + + '2424142^125.92^0^^72100'; + + const result = parseMessage(message); + + expect(result).toEqual([answer[0], answer[0]]); + }); +}); diff --git a/packages/backend/src/scraper/openapi/parse/openapi.parser.ts b/packages/backend/src/scraper/openapi/parse/openapi.parser.ts new file mode 100644 index 00000000..fe3e7002 --- /dev/null +++ b/packages/backend/src/scraper/openapi/parse/openapi.parser.ts @@ -0,0 +1,38 @@ +import { stockDataKeys } from '../type/openapiLiveData.type'; + +export const parseMessage = (data: string) => { + try { + return JSON.parse(data); + //eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return parseStockData(data); + } +}; +const FIELD_LENGTH: number = stockDataKeys.length; + +const parseStockData = (input: string) => { + const dataBlocks = input.split('|'); // 데이터 구분 + const results = []; + const size = parseInt(dataBlocks[2]); // 데이터 건수 + const rawData = dataBlocks[3]; + const values = rawData.split('^'); // 필드 구분자 '^' + + for (let i = 0; i < size; i++) { + //TODO : type narrowing require + const parsedData: Record = {}; + parsedData['STOCK_ID'] = values[i * FIELD_LENGTH]; + stockDataKeys.forEach((field: string, index: number) => { + const value = values[index + FIELD_LENGTH * i]; + if (!value) return (parsedData[field] = null); + + // 숫자형 필드 처리 + if (isNaN(parseInt(value))) { + parsedData[field] = value; // 문자열 그대로 저장 + } else { + parsedData[field] = parseFloat(value); // 숫자로 변환 + } + }); + results.push(parsedData); + } + return results; +}; diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts deleted file mode 100644 index 38015d48..00000000 --- a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts +++ /dev/null @@ -1,214 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any*/ -/* eslint-disable max-lines-per-function */ - -export type DetailDataQuery = { - fid_cond_mrkt_div_code: 'J'; - fid_input_iscd: string; - fid_div_cls_code: '0' | '1'; -}; - -export type FinancialRatio = { - stac_yymm: string; // 결산 년월 - grs: string; // 매출액 증가율 - bsop_prfi_inrt: string; // 영업 이익 증가율 - ntin_inrt: string; // 순이익 증가율 - roe_val: string; // ROE 값 - eps: string; // EPS - sps: string; // 주당매출액 - bps: string; // BPS - rsrv_rate: string; // 유보 비율 - lblt_rate: string; // 부채 비율 -}; - -export function isFinancialRatioData(data: any): data is FinancialRatio { - return ( - data && - typeof data.stac_yymm === 'string' && - typeof data.grs === 'string' && - typeof data.bsop_prfi_inrt === 'string' && - typeof data.ntin_inrt === 'string' && - typeof data.roe_val === 'string' && - typeof data.eps === 'string' && - typeof data.sps === 'string' && - typeof data.bps === 'string' && - typeof data.rsrv_rate === 'string' && - typeof data.lblt_rate === 'string' - ); -} - -export type ProductDetail = { - pdno: string; // 상품번호 - prdt_type_cd: string; // 상품유형코드 - mket_id_cd: string; // 시장ID코드 - scty_grp_id_cd: string; // 증권그룹ID코드 - excg_dvsn_cd: string; // 거래소구분코드 - setl_mmdd: string; // 결산월일 - lstg_stqt: string; // 상장주수 - 이거 사용 - lstg_cptl_amt: string; // 상장자본금액 - cpta: string; // 자본금 - papr: string; // 액면가 - issu_pric: string; // 발행가격 - kospi200_item_yn: string; // 코스피200종목여부 - 이것도 사용 - scts_mket_lstg_dt: string; // 유가증권시장상장일자 - scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 - kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 - kosdaq_mket_lstg_abol_dt: string; // 코스닥시장상장폐지일자 - frbd_mket_lstg_dt: string; // 프리보드시장상장일자 - frbd_mket_lstg_abol_dt: string; // 프리보드시장상장폐지일자 - reits_kind_cd: string; // 리츠종류코드 - etf_dvsn_cd: string; // ETF구분코드 - oilf_fund_yn: string; // 유전펀드여부 - idx_bztp_lcls_cd: string; // 지수업종대분류코드 - idx_bztp_mcls_cd: string; // 지수업종중분류코드 - idx_bztp_scls_cd: string; // 지수업종소분류코드 - stck_kind_cd: string; // 주식종류코드 - mfnd_opng_dt: string; // 뮤추얼펀드개시일자 - mfnd_end_dt: string; // 뮤추얼펀드종료일자 - dpsi_erlm_cncl_dt: string; // 예탁등록취소일자 - etf_cu_qty: string; // ETFCU수량 - prdt_name: string; // 상품명 - prdt_name120: string; // 상품명120 - prdt_abrv_name: string; // 상품약어명 - std_pdno: string; // 표준상품번호 - prdt_eng_name: string; // 상품영문명 - prdt_eng_name120: string; // 상품영문명120 - prdt_eng_abrv_name: string; // 상품영문약어명 - dpsi_aptm_erlm_yn: string; // 예탁지정등록여부 - etf_txtn_type_cd: string; // ETF과세유형코드 - etf_type_cd: string; // ETF유형코드 - lstg_abol_dt: string; // 상장폐지일자 - nwst_odst_dvsn_cd: string; // 신주구주구분코드 - sbst_pric: string; // 대용가격 - thco_sbst_pric: string; // 당사대용가격 - thco_sbst_pric_chng_dt: string; // 당사대용가격변경일자 - tr_stop_yn: string; // 거래정지여부 - admn_item_yn: string; // 관리종목여부 - thdt_clpr: string; // 당일종가 - bfdy_clpr: string; // 전일종가 - clpr_chng_dt: string; // 종가변경일자 - std_idst_clsf_cd: string; // 표준산업분류코드 - std_idst_clsf_cd_name: string; // 표준산업분류코드명 - idx_bztp_lcls_cd_name: string; // 지수업종대분류코드명 - idx_bztp_mcls_cd_name: string; // 지수업종중분류코드명 - idx_bztp_scls_cd_name: string; // 지수업종소분류코드명 - ocr_no: string; // OCR번호 - crfd_item_yn: string; // 크라우드펀딩종목여부 - elec_scty_yn: string; // 전자증권여부 - issu_istt_cd: string; // 발행기관코드 - etf_chas_erng_rt_dbnb: string; // ETF추적수익율배수 - etf_etn_ivst_heed_item_yn: string; // ETFETN투자유의종목여부 - stln_int_rt_dvsn_cd: string; // 대주이자율구분코드 - frnr_psnl_lmt_rt: string; // 외국인개인한도비율 - lstg_rqsr_issu_istt_cd: string; // 상장신청인발행기관코드 - lstg_rqsr_item_cd: string; // 상장신청인종목코드 - trst_istt_issu_istt_cd: string; // 신탁기관발행기관코드 -}; - -export const isProductDetail = (data: any): data is ProductDetail => { - return ( - typeof data.pdno === 'string' && - typeof data.prdt_type_cd === 'string' && - typeof data.mket_id_cd === 'string' && - typeof data.scty_grp_id_cd === 'string' && - typeof data.excg_dvsn_cd === 'string' && - typeof data.setl_mmdd === 'string' && - typeof data.lstg_stqt === 'string' && - typeof data.lstg_cptl_amt === 'string' && - typeof data.cpta === 'string' && - typeof data.papr === 'string' && - typeof data.issu_pric === 'string' && - typeof data.kospi200_item_yn === 'string' && - typeof data.scts_mket_lstg_dt === 'string' && - typeof data.scts_mket_lstg_abol_dt === 'string' && - typeof data.kosdaq_mket_lstg_dt === 'string' && - typeof data.kosdaq_mket_lstg_abol_dt === 'string' && - typeof data.frbd_mket_lstg_dt === 'string' && - typeof data.frbd_mket_lstg_abol_dt === 'string' && - typeof data.reits_kind_cd === 'string' && - typeof data.etf_dvsn_cd === 'string' && - typeof data.oilf_fund_yn === 'string' && - typeof data.idx_bztp_lcls_cd === 'string' && - typeof data.idx_bztp_mcls_cd === 'string' && - typeof data.idx_bztp_scls_cd === 'string' && - typeof data.stck_kind_cd === 'string' && - typeof data.mfnd_opng_dt === 'string' && - typeof data.mfnd_end_dt === 'string' && - typeof data.dpsi_erlm_cncl_dt === 'string' && - typeof data.etf_cu_qty === 'string' && - typeof data.prdt_name === 'string' && - typeof data.prdt_name120 === 'string' && - typeof data.prdt_abrv_name === 'string' && - typeof data.std_pdno === 'string' && - typeof data.prdt_eng_name === 'string' && - typeof data.prdt_eng_name120 === 'string' && - typeof data.prdt_eng_abrv_name === 'string' && - typeof data.dpsi_aptm_erlm_yn === 'string' && - typeof data.etf_txtn_type_cd === 'string' && - typeof data.etf_type_cd === 'string' && - typeof data.lstg_abol_dt === 'string' && - typeof data.nwst_odst_dvsn_cd === 'string' && - typeof data.sbst_pric === 'string' && - typeof data.thco_sbst_pric === 'string' && - typeof data.thco_sbst_pric_chng_dt === 'string' && - typeof data.tr_stop_yn === 'string' && - typeof data.admn_item_yn === 'string' && - typeof data.thdt_clpr === 'string' && - typeof data.bfdy_clpr === 'string' && - typeof data.clpr_chng_dt === 'string' && - typeof data.std_idst_clsf_cd === 'string' && - typeof data.std_idst_clsf_cd_name === 'string' && - typeof data.idx_bztp_lcls_cd_name === 'string' && - typeof data.idx_bztp_mcls_cd_name === 'string' && - typeof data.idx_bztp_scls_cd_name === 'string' && - typeof data.ocr_no === 'string' && - typeof data.crfd_item_yn === 'string' && - typeof data.elec_scty_yn === 'string' && - typeof data.issu_istt_cd === 'string' && - typeof data.etf_chas_erng_rt_dbnb === 'string' && - typeof data.etf_etn_ivst_heed_item_yn === 'string' && - typeof data.stln_int_rt_dvsn_cd === 'string' && - typeof data.frnr_psnl_lmt_rt === 'string' && - typeof data.lstg_rqsr_issu_istt_cd === 'string' && - typeof data.lstg_rqsr_item_cd === 'string' && - typeof data.trst_istt_issu_istt_cd === 'string' - ); -}; - -export type StockDetailQuery = { - pdno: string; - prdt_type_cd: string; -}; - -//export type FinancialDetail = { -// stac_yymm: string; // 결산 년월 -// sale_account: string; // 매출액 -// sale_cost: string; // 매출원가 -// sale_totl_prfi: string; // 매출총이익 -// depr_cost: string; // 감가상각비 -// sell_mang: string; // 판매관리비 -// bsop_prti: string; // 영업이익 -// bsop_non_ernn: string; // 영업외수익 -// bsop_non_expn: string; // 영업외비용 -// op_prfi: string; // 영업이익 -// spec_prfi: string; // 특별이익 -// spec_loss: string; // 특별손실 -// thtr_ntin: string; // 세전순이익 -//}; - -//export const isFinancialDetail = (data: any): data is FinancialDetail => { -// return ( -// typeof data.stac_yymm === 'string' && -// typeof data.sale_account === 'string' && -// typeof data.sale_cost === 'string' && -// typeof data.sale_totl_prfi === 'string' && -// typeof data.depr_cost === 'string' && -// typeof data.sell_mang === 'string' && -// typeof data.bsop_prti === 'string' && -// typeof data.bsop_non_ernn === 'string' && -// typeof data.bsop_non_expn === 'string' && -// typeof data.op_prfi === 'string' && -// typeof data.spec_prfi === 'string' && -// typeof data.spec_loss === 'string' && -// typeof data.thtr_ntin === 'string' -// ); -//}; diff --git a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts deleted file mode 100644 index d8041e7b..00000000 --- a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable max-lines-per-function */ - -export type StockData = { - MKSC_SHRN_ISCD: string; // 유가증권 단축 종목코드 - STCK_CNTG_HOUR: string; // 주식 체결 시간 - STCK_PRPR: string; // 주식 현재가 - PRDY_VRSS_SIGN: string; // 전일 대비 부호 - PRDY_VRSS: string; // 전일 대비 - PRDY_CTRT: string; // 전일 대비율 - WGHN_AVRG_STCK_PRC: string; // 가중 평균 주식 가격 - STCK_OPRC: string; // 주식 시가 - STCK_HGPR: string; // 주식 최고가 - STCK_LWPR: string; // 주식 최저가 - ASKP1: string; // 매도호가1 - BIDP1: string; // 매수호가1 - CNTG_VOL: string; // 체결 거래량 - ACML_VOL: string; // 누적 거래량 - ACML_TR_PBMN: string; // 누적 거래 대금 - SELN_CNTG_CSNU: string; // 매도 체결 건수 - SHNU_CNTG_CSNU: string; // 매수 체결 건수 - NTBY_CNTG_CSNU: string; // 순매수 체결 건수 - CTTR: string; // 체결강도 - SELN_CNTG_SMTN: string; // 총 매도 수량 - SHNU_CNTG_SMTN: string; // 총 매수 수량 - CCLD_DVSN: string; // 체결구분 - SHNU_RATE: string; // 매수비율 - PRDY_VOL_VRSS_ACML_VOL_RATE: string; // 전일 거래량 대비 등락율 - OPRC_HOUR: string; // 시가 시간 - OPRC_VRSS_PRPR_SIGN: string; // 시가대비구분 - OPRC_VRSS_PRPR: string; // 시가대비 - HGPR_HOUR: string; // 최고가 시간 - HGPR_VRSS_PRPR_SIGN: string; // 고가대비구분 - HGPR_VRSS_PRPR: string; // 고가대비 - LWPR_HOUR: string; // 최저가 시간 - LWPR_VRSS_PRPR_SIGN: string; // 저가대비구분 - LWPR_VRSS_PRPR: string; // 저가대비 - BSOP_DATE: string; // 영업 일자 - NEW_MKOP_CLS_CODE: string; // 신 장운영 구분 코드 - TRHT_YN: string; // 거래정지 여부 - ASKP_RSQN1: string; // 매도호가 잔량1 - BIDP_RSQN1: string; // 매수호가 잔량1 - TOTAL_ASKP_RSQN: string; // 총 매도호가 잔량 - TOTAL_BIDP_RSQN: string; // 총 매수호가 잔량 - VOL_TNRT: string; // 거래량 회전율 - PRDY_SMNS_HOUR_ACML_VOL: string; // 전일 동시간 누적 거래량 - PRDY_SMNS_HOUR_ACML_VOL_RATE: string; // 전일 동시간 누적 거래량 비율 - HOUR_CLS_CODE: string; // 시간 구분 코드 - MRKT_TRTM_CLS_CODE: string; // 임의종료구분코드 - VI_STND_PRC: string; // 정적VI발동기준가 -}; - -export function parseStockData(message: string[]): StockData { - return { - MKSC_SHRN_ISCD: message[0], - STCK_CNTG_HOUR: message[1], - STCK_PRPR: message[2], - PRDY_VRSS_SIGN: message[3], - PRDY_VRSS: message[4], - PRDY_CTRT: message[5], - WGHN_AVRG_STCK_PRC: message[6], - STCK_OPRC: message[7], - STCK_HGPR: message[8], - STCK_LWPR: message[9], - ASKP1: message[10], - BIDP1: message[11], - CNTG_VOL: message[12], - ACML_VOL: message[13], - ACML_TR_PBMN: message[14], - SELN_CNTG_CSNU: message[15], - SHNU_CNTG_CSNU: message[16], - NTBY_CNTG_CSNU: message[17], - CTTR: message[18], - SELN_CNTG_SMTN: message[19], - SHNU_CNTG_SMTN: message[20], - CCLD_DVSN: message[21], - SHNU_RATE: message[22], - PRDY_VOL_VRSS_ACML_VOL_RATE: message[23], - OPRC_HOUR: message[24], - OPRC_VRSS_PRPR_SIGN: message[25], - OPRC_VRSS_PRPR: message[26], - HGPR_HOUR: message[27], - HGPR_VRSS_PRPR_SIGN: message[28], - HGPR_VRSS_PRPR: message[29], - LWPR_HOUR: message[30], - LWPR_VRSS_PRPR_SIGN: message[31], - LWPR_VRSS_PRPR: message[32], - BSOP_DATE: message[33], - NEW_MKOP_CLS_CODE: message[34], - TRHT_YN: message[35], - ASKP_RSQN1: message[36], - BIDP_RSQN1: message[37], - TOTAL_ASKP_RSQN: message[38], - TOTAL_BIDP_RSQN: message[39], - VOL_TNRT: message[40], - PRDY_SMNS_HOUR_ACML_VOL: message[41], - PRDY_SMNS_HOUR_ACML_VOL_RATE: message[42], - HOUR_CLS_CODE: message[43], - MRKT_TRTM_CLS_CODE: message[44], - VI_STND_PRC: message[45], - }; -} - -export type OpenApiMessage = { - header: { - approval_key: string; - custtype: string; - tr_type: string; - 'content-type': string; - }; - body: { - input: { - tr_id: string; - tr_key: string; - }; - }; -}; - -export type MessageResponse = { - header: { - tr_id: string; - tr_key: string; - encrypt: string; - }; - body: { - rt_cd: string; - msg_cd: string; - msg1: string; - output?: { - iv: string; - key: string; - }; - }; -}; - -export function isMessageResponse(data: any): data is MessageResponse { - return ( - typeof data === 'object' && - data !== null && - typeof data.header === 'object' && - data.header !== null && - typeof data.header.tr_id === 'object' && - typeof data.header.tr_key === 'object' && - typeof data.header.encrypt === 'object' && - typeof data.body === 'object' && - data.body !== null && - typeof data.body.rt_cd === 'object' && - typeof data.body.msg_cd === 'object' && - typeof data.body.msg1 === 'object' && - typeof data.body.output === 'object' - ); -} diff --git a/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts b/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts deleted file mode 100644 index 5deb2d9e..00000000 --- a/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -export type MinuteData = { - stck_bsop_date: string; - stck_cntg_hour: string; - stck_prpr: string; - stck_oprc: string; - stck_hgpr: string; - stck_lwpr: string; - cntg_vol: string; - acml_tr_pbmn: string; -}; - -export type UpdateStockQuery = { - fid_etc_cls_code: string; - fid_cond_mrkt_div_code: 'J' | 'W'; - fid_input_iscd: string; - fid_input_hour_1: string; - fid_pw_data_incu_yn: 'Y' | 'N'; -}; - -export const isMinuteData = (data: any) => { - return ( - typeof data.stck_bsop_date === 'string' && - typeof data.stck_cntg_hour === 'string' && - typeof data.stck_prpr === 'string' && - typeof data.stck_oprc === 'string' && - typeof data.stck_hgpr === 'string' && - typeof data.stck_lwpr === 'string' && - typeof data.cntg_vol === 'string' && - typeof data.acml_tr_pbmn === 'string' - ); -}; diff --git a/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts b/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts deleted file mode 100644 index 4acc7f44..00000000 --- a/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -export type Period = 'D' | 'W' | 'M' | 'Y'; -export type ChartData = { - stck_bsop_date: string; - stck_clpr: string; - stck_oprc: string; - stck_hgpr: string; - stck_lwpr: string; - acml_vol: string; - acml_tr_pbmn: string; - flng_cls_code: string; - prtt_rate: string; - mod_yn: string; - prdy_vrss_sign: string; - prdy_vrss: string; - revl_issu_reas: string; -}; - -export type ItemChartPriceQuery = { - fid_cond_mrkt_div_code: 'J' | 'W'; - fid_input_iscd: string; - fid_input_date_1: string; - fid_input_date_2: string; - fid_period_div_code: Period; - fid_org_adj_prc: number; -}; - -export const isChartData = (data: any) => { - return ( - typeof data.stck_bsop_date === 'string' && - typeof data.stck_clpr === 'string' && - typeof data.stck_oprc === 'string' && - typeof data.stck_hgpr === 'string' && - typeof data.stck_lwpr === 'string' && - typeof data.acml_vol === 'string' && - typeof data.acml_tr_pbmn === 'string' && - typeof data.flng_cls_code === 'string' && - typeof data.prtt_rate === 'string' && - typeof data.mod_yn === 'string' && - typeof data.prdy_vrss_sign === 'string' && - typeof data.prdy_vrss === 'string' && - typeof data.revl_issu_reas === 'string' - ); -}; diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts deleted file mode 100644 index 6df0ca19..00000000 --- a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts +++ /dev/null @@ -1,13 +0,0 @@ -export type TR_ID = - | 'FHKST03010100' - | 'FHKST03010200' - | 'FHKST66430300' - | 'HHKDB669107C0' - | 'CTPF1002R'; - -export const TR_IDS: Record = { - ITEM_CHART_PRICE: 'FHKST03010100', - MINUTE_DATA: 'FHKST03010200', - FINANCIAL_DATA: 'FHKST66430300', - PRODUCTION_DETAIL: 'CTPF1002R', -}; diff --git a/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts b/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts deleted file mode 100644 index 1e0c3913..00000000 --- a/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; - -export class OpenapiException extends HttpException { - private error: unknown; - constructor(message: string, status: HttpStatus, error?: unknown) { - super(message, status); - this.error = error; - } - - public getError() { - return this.error; - } -} diff --git a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts deleted file mode 100644 index fa8f75b4..00000000 --- a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any*/ -import * as crypto from 'crypto'; -import { HttpStatus } from '@nestjs/common'; -import axios from 'axios'; -import { openApiConfig } from '../config/openapi.config'; -import { TR_ID } from '../type/openapiUtil.type'; -import { OpenapiException } from './openapiCustom.error'; - -const throwOpenapiException = (error: any) => { - if (error.message && error.response && error.response.status) { - throw new OpenapiException( - `Request failed: ${error.message}`, - error.response.status, - error, - ); - } else { - throw new OpenapiException( - `Unknown error: ${error.message || 'No message'}`, - HttpStatus.INTERNAL_SERVER_ERROR, - error, - ); - } -}; - -const postOpenApi = async ( - url: string, - config: typeof openApiConfig, - body: object, -) => { - try { - const response = await axios.post(config.STOCK_URL + url, body); - return response.data; - } catch (error) { - throwOpenapiException(error); - } -}; - -const getOpenApi = async ( - url: string, - config: typeof openApiConfig, - query: object, - tr_id: TR_ID, -) => { - try { - const response = await axios.get(config.STOCK_URL + url, { - params: query, - headers: { - Authorization: `Bearer ${config.STOCK_API_TOKEN}`, - appkey: config.STOCK_API_KEY, - appsecret: config.STOCK_API_PASSWORD, - tr_id, - custtype: 'P', - }, - }); - return response.data; - } catch (error) { - throwOpenapiException(error); - } -}; - -const getTodayDate = (): string => { - const today = new Date(); - return today.toISOString().split('T')[0].replace(/-/g, ''); -}; - -const getPreviousDate = (date: string, months: number): string => { - const currentDate = new Date( - date.slice(0, 4) + '-' + date.slice(4, 6) + '-' + date.slice(6, 8), - ); - currentDate.setMonth(currentDate.getMonth() - months); - return currentDate.toISOString().split('T')[0].replace(/-/g, ''); -}; - -const getCurrentTime = () => { - const now = new Date(); - const hours = String(now.getHours()).padStart(2, '0'); - const minutes = String(now.getMinutes()).padStart(2, '0'); - const seconds = String(now.getSeconds()).padStart(2, '0'); - return `${hours}${minutes}${seconds}`; -}; - -const decryptAES256 = ( - encryptedText: string, - key: string, - iv: string, -): string => { - const decipher = crypto.createDecipheriv( - 'aes-256-cbc', - Buffer.from(key, 'hex'), - Buffer.from(iv, 'hex'), - ); - let decrypted = decipher.update(encryptedText, 'hex', 'utf8'); - decrypted += decipher.final('utf8'); - return decrypted; -}; - -const bufferToObject = (buffer: Buffer): any => { - try { - const jsonString = buffer.toString('utf-8'); - return JSON.parse(jsonString); - } catch (error) { - console.error('Failed to convert buffer to object:', error); - throw error; - } -}; - -export { - postOpenApi, - getOpenApi, - getTodayDate, - getPreviousDate, - getCurrentTime, - decryptAES256, - bufferToObject, -}; diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts deleted file mode 100644 index 554c3eff..00000000 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Inject, Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Logger } from 'winston'; -import { WebSocket } from 'ws'; -import { OpenapiLiveData } from './api/openapiLiveData.api'; - -@Injectable() -export class WebsocketClient { - private client: WebSocket; - private readonly reconnectInterval = 60000; - private readonly url = - process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; - - constructor( - @Inject('winston') private readonly logger: Logger, - private readonly openapiLiveData: OpenapiLiveData, - ) { - if (process.env.NODE_ENV === 'production') { - this.connect(); - } - } - - // TODO : subscribe 구조로 리팩토링 - private subscribe() {} - - private message(data: any) { - this.logger.info(`Received message: ${data}`); - if (data.header && data.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.sendPong(); - return; - } - if (data.header && data.header.tr_id === 'H0STCNT0') { - return; - } - this.openapiLiveData.output(data); - } - - @Cron('0 2 * * 1-5') - private connect() { - this.client = new WebSocket(this.url); - - this.client.on('open', () => { - this.logger.info('WebSocket connection established'); - this.openapiLiveData.getMessage().then((val) => { - val.forEach((message) => this.sendMessage(message)); - }); - }); - - this.client.on('message', (data: any) => { - try { - this.message(data); - } catch (error) { - this.logger.info(error); - } - }); - - this.client.on('close', () => { - this.logger.warn( - `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, - ); - setTimeout(() => this.connect(), this.reconnectInterval); - }); - - this.client.on('error', (error: any) => { - this.logger.error(`WebSocket error: ${error.message}`); - }); - } - - private sendPong() { - const pongMessage = { - header: { tr_id: 'PINGPONG', datetime: new Date().toISOString() }, - }; - this.client.send(JSON.stringify(pongMessage)); - this.logger.info(`Sent PONG: ${JSON.stringify(pongMessage)}`); - } - - private sendMessage(message: string) { - if (this.client.readyState === WebSocket.OPEN) { - this.client.send(message); - this.logger.info(`Sent message: ${message}`); - } else { - this.logger.warn('WebSocket is not open. Message not sent.'); - } - } -} From a188a032b7c293aa52be4c1a77479a9bd3819de6 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 21:11:02 +0900 Subject: [PATCH 083/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20websoc?= =?UTF-8?q?ket=20client=20service=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../korea-stock-info/openapi/websocketClient.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts index 3073683e..2e36acfb 100644 --- a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts @@ -4,11 +4,12 @@ import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; import { RawData, WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; +import { openApiToken } from './api/openapiToken.api'; +import { openApiConfig } from './config/openapi.config'; import { parseMessage } from './parse/openapi.parser'; -import { openApiToken } from '@/scraper/openapi/api/openapiToken.api'; -import { openApiConfig } from '@/scraper/openapi/config/openapi.config'; type TR_IDS = '0' | '1'; + @Injectable() export class WebsocketClient { private client: WebSocket; From df6709c62dee28a9f3671cf5ab38d6f52550a1ff Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 21:13:50 +0900 Subject: [PATCH 084/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20api=20?= =?UTF-8?q?=ED=8F=B4=EB=8D=94=20=EC=9D=B4=EB=8F=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 279 ++++++++++++++++++ .../openapi/api/openapiLiveData.api.ts | 34 +++ .../openapi/api/openapiMinuteData.api.ts | 156 ++++++++++ .../openapi/api/openapiPeriodData.api.ts | 218 ++++++++++++++ .../scraper/openapi/api/openapiToken.api.ts | 112 +++++++ .../scraper/openapi/config/openapi.config.ts | 17 ++ .../scraper/openapi/openapi-scraper.module.ts | 43 +++ .../openapi/openapi-scraper.service.ts | 15 + .../openapi/type/openapiDetailData.type.ts | 214 ++++++++++++++ .../openapi/type/openapiLiveData.type.ts | 150 ++++++++++ .../openapi/type/openapiMinuteData.type.ts | 33 +++ .../scraper/openapi/type/openapiPeriodData.ts | 45 +++ .../scraper/openapi/type/openapiUtil.type.ts | 13 + .../openapi/util/openapiCustom.error.ts | 13 + .../scraper/openapi/util/openapiUtil.api.ts | 115 ++++++++ .../src/scraper/openapi/util/priorityQueue.ts | 99 +++++++ .../openapi/websocketClient.service.ts | 148 ++++++++++ 17 files changed, 1704 insertions(+) create mode 100644 packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts create mode 100644 packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts create mode 100644 packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts create mode 100644 packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts create mode 100644 packages/backend/src/scraper/openapi/api/openapiToken.api.ts create mode 100644 packages/backend/src/scraper/openapi/config/openapi.config.ts create mode 100644 packages/backend/src/scraper/openapi/openapi-scraper.module.ts create mode 100644 packages/backend/src/scraper/openapi/openapi-scraper.service.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiPeriodData.ts create mode 100644 packages/backend/src/scraper/openapi/type/openapiUtil.type.ts create mode 100644 packages/backend/src/scraper/openapi/util/openapiCustom.error.ts create mode 100644 packages/backend/src/scraper/openapi/util/openapiUtil.api.ts create mode 100644 packages/backend/src/scraper/openapi/util/priorityQueue.ts create mode 100644 packages/backend/src/scraper/openapi/websocketClient.service.ts diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts new file mode 100644 index 00000000..9be8e3ea --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -0,0 +1,279 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Between, DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { + DetailDataQuery, + FinancialRatio, + isFinancialRatioData, + isProductDetail, + ProductDetail, + StockDetailQuery, +} from '../type/openapiDetailData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { getOpenApi } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { KospiStock } from '@/stock/domain/kospiStock.entity'; +import { Stock } from '@/stock/domain/stock.entity'; +import { StockDaily } from '@/stock/domain/stockData.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; + +@Injectable() +export class OpenapiDetailData { + private readonly financialUrl: string = + '/uapi/domestic-stock/v1/finance/financial-ratio'; + private readonly productUrl: string = + '/uapi/domestic-stock/v1/quotations/search-stock-info'; + private readonly intervals = 1000; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + //setTimeout(() => this.getDetailData(), 5000); + } + + @Cron('0 8 * * 1-5') + async getDetailData() { + if (process.env.NODE_ENV !== 'production') return; + const entityManager = this.datasource.manager; + const stocks = await entityManager.find(Stock); + const configCount = openApiToken.configs.length; + const chunkSize = Math.ceil(stocks.length / configCount); + + for (let i = 0; i < configCount; i++) { + this.logger.info(openApiToken.configs[i]); + const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); + this.getDetailDataChunk(chunk, openApiToken.configs[i]); + } + } + + private async saveDetailData(stockDetail: StockDetail) { + const manager = this.datasource.manager; + const entity = StockDetail; + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } + } + + private async saveKospiData(stockDetail: KospiStock) { + const manager = this.datasource.manager; + const entity = KospiStock; + const existingStockDetail = await manager.findOne(entity, { + where: { + stock: { id: stockDetail.stock.id }, + }, + }); + + if (existingStockDetail) { + manager.update( + entity, + { stock: { id: stockDetail.stock.id } }, + stockDetail, + ); + } else { + manager.save(entity, stockDetail); + } + } + + private async calPer(eps: number): Promise { + if (eps <= 0) return NaN; + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const per = currentPrice / eps; + + if (isNaN(per)) return 0; + else return per; + } else { + return 0; + } + } + + private async calMarketCap(lstg: number) { + const manager = this.datasource.manager; + const latestResult = await manager.find(StockDaily, { + skip: 0, + take: 1, + order: { createdAt: 'desc' }, + }); + + // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 + if (latestResult && latestResult[0] && latestResult[0].close) { + const currentPrice = latestResult[0].close; + const marketCap = lstg * currentPrice; + + if (isNaN(marketCap)) return 0; + else return marketCap; + } else { + return 0; + } + } + + private async get52WeeksLowHigh() { + const manager = this.datasource.manager; + const nowDate = new Date(); + const weeksAgoDate = this.getDate52WeeksAgo(); + // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 + const output = await manager.find(StockDaily, { + select: ['low', 'high'], + where: { + startTime: Between(weeksAgoDate, nowDate), + }, + }); + const result = output.reduce((prev, cur) => { + if (prev.low > cur.low) prev.low = cur.low; + if (prev.high < cur.high) prev.high = cur.high; + return cur; + }, new StockDaily()); + let low = 0; + let high = 0; + if (result.low && !isNaN(result.low)) low = result.low; + if (result.high && !isNaN(result.high)) high = result.high; + return { low, high }; + } + + private async makeStockDetailObject( + output1: FinancialRatio, + output2: ProductDetail, + stockId: string, + ): Promise { + const result = new StockDetail(); + result.stock = { id: stockId } as Stock; + result.marketCap = + (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; + result.eps = parseInt(output1.eps); + const { low, high } = await this.get52WeeksLowHigh(); + result.low52w = low; + result.high52w = high; + const eps = parseInt(output1.eps); + if (isNaN(eps)) result.eps = 0; + else result.eps = eps; + const per = await this.calPer(eps); + if (isNaN(per)) result.per = 0; + else result.per = per; + result.updatedAt = new Date(); + return result; + } + + private async makeKospiStockObject(output: ProductDetail, stockId: string) { + const ret = new KospiStock(); + ret.isKospi = output.kospi200_item_yn === 'Y' ? true : false; + ret.stock = { id: stockId } as Stock; + return ret; + } + + private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { + const dataQuery = this.getDetailDataQuery(stock.id!); + // 여기서 가져올 건 eps -> eps와 per 계산하자. + try { + const response = await getOpenApi( + this.financialUrl, + conf, + dataQuery, + TR_IDS.FINANCIAL_DATA, + ); + if (response.output) { + const output1 = response.output; + return output1[0]; + } + } catch (error) { + this.logger.warn(error); + } + } + + private async getProductData(stock: Stock, conf: typeof openApiConfig) { + const defaultQuery = this.getFinancialDataQuery(stock.id!); + + // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 + try { + const response = await getOpenApi( + this.productUrl, + conf, + defaultQuery, + TR_IDS.PRODUCTION_DETAIL, + ); + if (response.output) { + const output2 = response.output; + return output2; + } + } catch (error) { + this.logger.warn(error); + } + } + + private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { + const output1 = await this.getFinancialRatio(stock, conf); + const output2 = await this.getProductData(stock, conf); + + this.logger.info(JSON.stringify(output1)); + this.logger.info(JSON.stringify(output2)); + if (isFinancialRatioData(output1) && isProductDetail(output2)) { + const stockDetail = await this.makeStockDetailObject( + output1, + output2, + stock.id!, + ); + this.saveDetailData(stockDetail); + const kospiStock = await this.makeKospiStockObject(output2, stock.id!); + this.saveKospiData(kospiStock); + + this.logger.info(`${stock.id!} is saved`); + } + } + + private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { + let delay = 0; + for await (const stock of chunk) { + setTimeout(() => this.getDetailDataDelay(stock, conf), delay); + delay += this.intervals; + } + } + + private getFinancialDataQuery( + stockId: string, + code: '300' | '301' | '302' | '306' = '300', + ): StockDetailQuery { + return { + pdno: stockId, + prdt_type_cd: code, + }; + } + + private getDetailDataQuery( + stockId: string, + divCode: 'J' = 'J', + classify: '0' | '1' = '0', + ): DetailDataQuery { + return { + fid_div_cls_code: classify, + fid_cond_mrkt_div_code: divCode, + fid_input_iscd: stockId, + }; + } + + private getDate52WeeksAgo(): Date { + const today = new Date(); + const weeksAgo = 52 * 7; + const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); + date52WeeksAgo.setHours(0, 0, 0, 0); + return date52WeeksAgo; + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts new file mode 100644 index 00000000..9729e0fc --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -0,0 +1,34 @@ +import { DataSource } from 'typeorm'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; + +export class OpenapiLiveData { + public readonly TR_ID: string = 'H0STCNT0'; + constructor(private readonly datasource: DataSource) {} + + async saveLiveData(data: StockLiveData[]) { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .insert() + .into(StockLiveData) + .values(data) + .execute(); + } + + convertLiveData(messages: Record[]): StockLiveData[] { + const stockData: StockLiveData[] = []; + messages.map((message) => { + const stockLiveData = new StockLiveData(); + stockLiveData.currentPrice = parseFloat(message.STCK_PRPR); + stockLiveData.changeRate = parseFloat(message.PRDY_CTRT); + stockLiveData.volume = parseInt(message.CNTG_VOL); + stockLiveData.high = parseFloat(message.STCK_HGPR); + stockLiveData.low = parseFloat(message.STCK_LWPR); + stockLiveData.open = parseFloat(message.STCK_OPRC); + stockLiveData.previousClose = parseFloat(message.WGHN_AVRG_STCK_PRC); + stockLiveData.updatedAt = new Date(); + stockData.push(stockLiveData); + }); + return stockData; + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts new file mode 100644 index 00000000..003e7560 --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -0,0 +1,156 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; + +import { + isMinuteData, + MinuteData, + UpdateStockQuery, +} from '../type/openapiMinuteData.type'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { Stock } from '@/stock/domain/stock.entity'; +import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; + +const STOCK_CUT = 4; + +@Injectable() +export class OpenapiMinuteData { + private stock: Stock[][] = []; + private readonly entity = StockMinutely; + private readonly url: string = + '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; + private readonly intervals: number = 130; + private flip: number = 0; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + this.getStockData(); + } + + @Cron('0 1 * * 1-5') + async getStockData() { + if (process.env.NODE_ENV !== 'production') return; + const stock = await this.datasource.manager.findBy(Stock, { + isTrading: true, + }); + const stockSize = Math.ceil(stock.length / STOCK_CUT); + let i = 0; + this.stock = []; + while (i < STOCK_CUT) { + this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); + i++; + } + } + + private convertResToMinuteData( + stockId: string, + item: MinuteData, + time: string, + ) { + const stockPeriod = new StockData(); + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(item.stck_bsop_date.slice(0, 4)), + parseInt(item.stck_bsop_date.slice(4, 6)) - 1, + parseInt(item.stck_bsop_date.slice(6, 8)), + parseInt(time.slice(0, 2)), + parseInt(time.slice(2, 4)), + ); + stockPeriod.close = parseInt(item.stck_prpr); + stockPeriod.open = parseInt(item.stck_oprc); + stockPeriod.high = parseInt(item.stck_hgpr); + stockPeriod.low = parseInt(item.stck_lwpr); + stockPeriod.volume = parseInt(item.cntg_vol); + stockPeriod.createdAt = new Date(); + return stockPeriod; + } + + private isMarketOpenTime(time: string) { + const numberTime = parseInt(time); + return numberTime >= 90000 && numberTime <= 153000; + } + + private async saveMinuteData( + stockId: string, + item: MinuteData[], + time: string, + ) { + const manager = this.datasource.manager; + if (!this.isMarketOpenTime(time)) return; + const stockPeriod = item.map((val) => + this.convertResToMinuteData(stockId, val, time), + ); + manager.save(this.entity, stockPeriod); + } + + private async getMinuteDataInterval( + stockId: string, + time: string, + config: typeof openApiConfig, + ) { + const query = this.getUpdateStockQuery(stockId, time); + try { + const response = await getOpenApi( + this.url, + config, + query, + TR_IDS.MINUTE_DATA, + ); + let output; + if (response.output2) output = response.output2; + if (output && output[0] && isMinuteData(output[0])) { + this.saveMinuteData(stockId, output, time); + } + } catch (error) { + this.logger.warn(error); + } + } + + private async getMinuteDataChunk( + chunk: Stock[], + config: typeof openApiConfig, + ) { + const time = getCurrentTime(); + let interval = 0; + for await (const stock of chunk) { + setTimeout( + () => this.getMinuteDataInterval(stock.id!, time, config), + interval, + ); + interval += this.intervals; + } + } + + @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) + getMinuteData() { + if (process.env.NODE_ENV !== 'production') return; + const configCount = openApiToken.configs.length; + const stock = this.stock[this.flip % STOCK_CUT]; + this.flip++; + const chunkSize = Math.ceil(stock.length / configCount); + for (let i = 0; i < configCount; i++) { + const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); + this.getMinuteDataChunk(chunk, openApiToken.configs[i]); + } + } + + private getUpdateStockQuery( + stockId: string, + time: string, + isPastData: boolean = true, + marketCode: 'J' | 'W' = 'J', + ): UpdateStockQuery { + return { + fid_etc_cls_code: '', + fid_cond_mrkt_div_code: marketCode, + fid_input_iscd: stockId, + fid_input_hour_1: time, + fid_pw_data_incu_yn: isPastData ? 'Y' : 'N', + }; + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts new file mode 100644 index 00000000..f4268088 --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -0,0 +1,218 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { DataSource, EntityManager } from 'typeorm'; +import { Logger } from 'winston'; +import { + ChartData, + isChartData, + ItemChartPriceQuery, + Period, +} from '../type/openapiPeriodData'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { + getOpenApi, + getPreviousDate, + getTodayDate, +} from '../util/openapiUtil.api'; +import { openApiToken } from './openapiToken.api'; +import { Stock } from '@/stock/domain/stock.entity'; +import { + StockData, + StockDaily, + StockWeekly, + StockMonthly, + StockYearly, +} from '@/stock/domain/stockData.entity'; + +const DATE_TO_ENTITY = { + D: StockDaily, + W: StockWeekly, + M: StockMonthly, + Y: StockYearly, +}; + +const DATE_TO_MONTH = { + D: 3, + W: 6, + M: 12, + Y: 24, +}; + +const INTERVALS = 4000; + +@Injectable() +export class OpenapiPeriodData { + private readonly url: string = + '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) { + //this.getItemChartPriceCheck(); + } + + @Cron('0 1 * * 1-5') + async getItemChartPriceCheck() { + if (process.env.NODE_ENV !== 'production') return; + const stocks = await this.datasource.manager.find(Stock, { + where: { + isTrading: true, + }, + }); + const configCount = openApiToken.configs.length; + const chunkSize = Math.ceil(stocks.length / configCount); + + for (let i = 0; i < configCount; i++) { + const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); + this.getChartData(chunk, 'D'); + setTimeout(() => this.getChartData(chunk, 'W'), INTERVALS); + setTimeout(() => this.getChartData(chunk, 'M'), INTERVALS * 2); + setTimeout(() => this.getChartData(chunk, 'Y'), INTERVALS * 3); + } + } + + private async getChartData(chunk: Stock[], period: Period) { + const baseTime = INTERVALS * 4; + const entity = DATE_TO_ENTITY[period]; + + let time = 0; + for (const stock of chunk) { + time += baseTime; + setTimeout(() => this.processStockData(stock, period, entity), time); + } + } + + private async processStockData( + stock: Stock, + period: Period, + entity: typeof StockData, + ) { + const stockPeriod = new StockData(); + const manager = this.datasource.manager; + let configIdx = 0; + let end = getTodayDate(); + let start = getPreviousDate(end, 3); + let isFail = false; + + while (!isFail) { + configIdx = (configIdx + 1) % openApiToken.configs.length; + this.setStockPeriod(stockPeriod, stock.id!, end); + + // chart 데이터가 있는 지 확인 -> 리턴 + if (await this.existsChartData(stockPeriod, manager, entity)) return; + + const query = this.getItemChartPriceQuery(stock.id!, start, end, period); + + const output = await this.fetchChartData(query, configIdx); + + if (output) { + await this.saveChartData(entity, stock.id!, output); + ({ endDate: end, startDate: start } = this.updateDates(start, period)); + } else isFail = true; + } + } + + private setStockPeriod( + stockPeriod: StockData, + stockId: string, + endDate: string, + ): void { + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(endDate.slice(0, 4)), + parseInt(endDate.slice(4, 6)) - 1, + parseInt(endDate.slice(6, 8)), + ); + } + + private async fetchChartData(query: ItemChartPriceQuery, configIdx: number) { + try { + const response = await getOpenApi( + this.url, + openApiToken.configs[configIdx], + query, + TR_IDS.ITEM_CHART_PRICE, + ); + return response.output2 as ChartData[]; + } catch (error) { + this.logger.warn(error); + } + } + + private updateDates( + startDate: string, + period: Period, + ): { endDate: string; startDate: string } { + const endDate = getPreviousDate(startDate, DATE_TO_MONTH[period]); + startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]); + return { endDate, startDate }; + } + + private async existsChartData( + stock: StockData, + manager: EntityManager, + entity: typeof StockData, + ) { + return await manager.findOne(entity, { + where: { + stock: { id: stock.stock.id }, + createdAt: stock.startTime, + }, + }); + } + + private async insertChartData(stock: StockData, entity: typeof StockData) { + const manager = this.datasource.manager; + if (!(await this.existsChartData(stock, manager, entity))) { + await manager.save(entity, stock); + } + } + + private convertObjectToStockData(item: ChartData, stockId: string) { + const stockPeriod = new StockData(); + stockPeriod.stock = { id: stockId } as Stock; + stockPeriod.startTime = new Date( + parseInt(item.stck_bsop_date.slice(0, 4)), + parseInt(item.stck_bsop_date.slice(4, 6)) - 1, + parseInt(item.stck_bsop_date.slice(6, 8)), + ); + stockPeriod.close = parseInt(item.stck_clpr); + stockPeriod.open = parseInt(item.stck_oprc); + stockPeriod.high = parseInt(item.stck_hgpr); + stockPeriod.low = parseInt(item.stck_lwpr); + stockPeriod.volume = parseInt(item.acml_vol); + stockPeriod.createdAt = new Date(); + return stockPeriod; + } + + private async saveChartData( + entity: typeof StockData, + stockId: string, + data: ChartData[], + ) { + for (const item of data) { + if (!isChartData(item)) { + continue; + } + const stockPeriod = this.convertObjectToStockData(item, stockId); + await this.insertChartData(stockPeriod, entity); + } + } + + private getItemChartPriceQuery( + stockId: string, + startDate: string, + endDate: string, + period: Period, + marketCode: 'J' | 'W' = 'J', + ): ItemChartPriceQuery { + return { + fid_cond_mrkt_div_code: marketCode, + fid_input_iscd: stockId, + fid_input_date_1: startDate, + fid_input_date_2: endDate, + fid_period_div_code: period, + fid_org_adj_prc: 0, + }; + } +} diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts new file mode 100644 index 00000000..6e6c88c5 --- /dev/null +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -0,0 +1,112 @@ +import { Inject } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { OpenapiException } from '../util/openapiCustom.error'; +import { postOpenApi } from '../util/openapiUtil.api'; +import { logger } from '@/configs/logger.config'; + +class OpenapiTokenApi { + private config: (typeof openApiConfig)[] = []; + constructor(@Inject('winston') private readonly logger: Logger) { + const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); + const api_keys = openApiConfig.STOCK_API_KEY!.split(','); + const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); + if ( + accounts.length === 0 || + accounts.length !== api_keys.length || + api_passwords.length !== api_keys.length + ) { + this.logger.warn('Open API Config Error'); + } + for (let i = 0; i < accounts.length; i++) { + this.config.push({ + STOCK_URL: openApiConfig.STOCK_URL, + STOCK_ACCOUNT: accounts[i], + STOCK_API_KEY: api_keys[i], + STOCK_API_PASSWORD: api_passwords[i], + }); + } + this.initAuthenValue(); + } + + get configs() { + //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 + return this.config; + } + + private async initAuthenValue() { + const delay = 60000; + const delayMinute = delay / 1000 / 60; + + try { + await this.initAccessToken(); + await this.initWebSocketKey(); + } catch (error) { + if (error instanceof Error) { + this.logger.warn( + `Request failed: ${error.message}. Retrying in ${delayMinute} minute...`, + ); + } else { + this.logger.warn( + `Request failed. Retrying in ${delayMinute} minute...`, + ); + setTimeout(async () => { + await this.initAccessToken(); + await this.initWebSocketKey(); + }, delay); + } + } + } + + @Cron('50 0 * * 1-5') + async initAccessToken() { + const updatedConfig = await Promise.all( + this.config.map(async (val) => { + val.STOCK_API_TOKEN = await this.getToken(val)!; + return val; + }), + ); + this.config = updatedConfig; + } + + @Cron('50 0 * * 1-5') + async initWebSocketKey() { + const updatedConfig = await Promise.all( + this.config.map(async (val) => { + val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; + return val; + }), + ); + this.config = updatedConfig; + } + + private async getToken(config: typeof openApiConfig): Promise { + const body = { + grant_type: 'client_credentials', + appkey: config.STOCK_API_KEY, + appsecret: config.STOCK_API_PASSWORD, + }; + const tmp = await postOpenApi('/oauth2/tokenP', config, body); + if (!tmp.access_token) { + throw new OpenapiException('Access Token Failed', 403); + } + return tmp.access_token as string; + } + + private async getWebSocketKey(config: typeof openApiConfig): Promise { + const body = { + grant_type: 'client_credentials', + appkey: config.STOCK_API_KEY, + secretkey: config.STOCK_API_PASSWORD, + }; + const tmp = await postOpenApi('/oauth2/Approval', config, body); + if (!tmp.approval_key) { + throw new OpenapiException('WebSocket Key Failed', 403); + } + return tmp.approval_key as string; + } +} + +const openApiToken = new OpenapiTokenApi(logger); +export { openApiToken }; diff --git a/packages/backend/src/scraper/openapi/config/openapi.config.ts b/packages/backend/src/scraper/openapi/config/openapi.config.ts new file mode 100644 index 00000000..8aa12ea3 --- /dev/null +++ b/packages/backend/src/scraper/openapi/config/openapi.config.ts @@ -0,0 +1,17 @@ +import * as dotenv from 'dotenv'; + +dotenv.config(); + +export const openApiConfig: { + STOCK_URL: string | undefined; + STOCK_ACCOUNT: string | undefined; + STOCK_API_KEY: string | undefined; + STOCK_API_PASSWORD: string | undefined; + STOCK_API_TOKEN?: string; + STOCK_WEBSOCKET_KEY?: string; +} = { + STOCK_URL: process.env.STOCK_URL, + STOCK_ACCOUNT: process.env.STOCK_ACCOUNT, + STOCK_API_KEY: process.env.STOCK_API_KEY, + STOCK_API_PASSWORD: process.env.STOCK_API_PASSWORD, +}; diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts new file mode 100644 index 00000000..cb45c91c --- /dev/null +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -0,0 +1,43 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; +import { OpenapiMinuteData } from './api/openapiMinuteData.api'; +import { OpenapiPeriodData } from './api/openapiPeriodData.api'; +import { OpenapiScraperService } from './openapi-scraper.service'; +import { WebsocketClient } from './websocketClient.service'; +import { Stock } from '@/stock/domain/stock.entity'; +import { + StockDaily, + StockMinutely, + StockMonthly, + StockWeekly, + StockYearly, +} from '@/stock/domain/stockData.entity'; +import { StockDetail } from '@/stock/domain/stockDetail.entity'; +import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + Stock, + StockMinutely, + StockDaily, + StockWeekly, + StockMonthly, + StockYearly, + StockLiveData, + StockDetail, + ]), + ], + controllers: [], + providers: [ + OpenapiPeriodData, + OpenapiMinuteData, + OpenapiDetailData, + OpenapiScraperService, + OpenapiLiveData, + WebsocketClient, + ], +}) +export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts new file mode 100644 index 00000000..52c90179 --- /dev/null +++ b/packages/backend/src/scraper/openapi/openapi-scraper.service.ts @@ -0,0 +1,15 @@ +import { Injectable } from '@nestjs/common'; +import { DataSource } from 'typeorm'; +import { OpenapiDetailData } from './api/openapiDetailData.api'; +import { OpenapiMinuteData } from './api/openapiMinuteData.api'; +import { OpenapiPeriodData } from './api/openapiPeriodData.api'; + +@Injectable() +export class OpenapiScraperService { + public constructor( + private datasource: DataSource, + private readonly openapiPeriodData: OpenapiPeriodData, + private readonly openapiMinuteData: OpenapiMinuteData, + private readonly openapiDetailData: OpenapiDetailData, + ) {} +} diff --git a/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts new file mode 100644 index 00000000..38015d48 --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiDetailData.type.ts @@ -0,0 +1,214 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +/* eslint-disable max-lines-per-function */ + +export type DetailDataQuery = { + fid_cond_mrkt_div_code: 'J'; + fid_input_iscd: string; + fid_div_cls_code: '0' | '1'; +}; + +export type FinancialRatio = { + stac_yymm: string; // 결산 년월 + grs: string; // 매출액 증가율 + bsop_prfi_inrt: string; // 영업 이익 증가율 + ntin_inrt: string; // 순이익 증가율 + roe_val: string; // ROE 값 + eps: string; // EPS + sps: string; // 주당매출액 + bps: string; // BPS + rsrv_rate: string; // 유보 비율 + lblt_rate: string; // 부채 비율 +}; + +export function isFinancialRatioData(data: any): data is FinancialRatio { + return ( + data && + typeof data.stac_yymm === 'string' && + typeof data.grs === 'string' && + typeof data.bsop_prfi_inrt === 'string' && + typeof data.ntin_inrt === 'string' && + typeof data.roe_val === 'string' && + typeof data.eps === 'string' && + typeof data.sps === 'string' && + typeof data.bps === 'string' && + typeof data.rsrv_rate === 'string' && + typeof data.lblt_rate === 'string' + ); +} + +export type ProductDetail = { + pdno: string; // 상품번호 + prdt_type_cd: string; // 상품유형코드 + mket_id_cd: string; // 시장ID코드 + scty_grp_id_cd: string; // 증권그룹ID코드 + excg_dvsn_cd: string; // 거래소구분코드 + setl_mmdd: string; // 결산월일 + lstg_stqt: string; // 상장주수 - 이거 사용 + lstg_cptl_amt: string; // 상장자본금액 + cpta: string; // 자본금 + papr: string; // 액면가 + issu_pric: string; // 발행가격 + kospi200_item_yn: string; // 코스피200종목여부 - 이것도 사용 + scts_mket_lstg_dt: string; // 유가증권시장상장일자 + scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 + kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 + kosdaq_mket_lstg_abol_dt: string; // 코스닥시장상장폐지일자 + frbd_mket_lstg_dt: string; // 프리보드시장상장일자 + frbd_mket_lstg_abol_dt: string; // 프리보드시장상장폐지일자 + reits_kind_cd: string; // 리츠종류코드 + etf_dvsn_cd: string; // ETF구분코드 + oilf_fund_yn: string; // 유전펀드여부 + idx_bztp_lcls_cd: string; // 지수업종대분류코드 + idx_bztp_mcls_cd: string; // 지수업종중분류코드 + idx_bztp_scls_cd: string; // 지수업종소분류코드 + stck_kind_cd: string; // 주식종류코드 + mfnd_opng_dt: string; // 뮤추얼펀드개시일자 + mfnd_end_dt: string; // 뮤추얼펀드종료일자 + dpsi_erlm_cncl_dt: string; // 예탁등록취소일자 + etf_cu_qty: string; // ETFCU수량 + prdt_name: string; // 상품명 + prdt_name120: string; // 상품명120 + prdt_abrv_name: string; // 상품약어명 + std_pdno: string; // 표준상품번호 + prdt_eng_name: string; // 상품영문명 + prdt_eng_name120: string; // 상품영문명120 + prdt_eng_abrv_name: string; // 상품영문약어명 + dpsi_aptm_erlm_yn: string; // 예탁지정등록여부 + etf_txtn_type_cd: string; // ETF과세유형코드 + etf_type_cd: string; // ETF유형코드 + lstg_abol_dt: string; // 상장폐지일자 + nwst_odst_dvsn_cd: string; // 신주구주구분코드 + sbst_pric: string; // 대용가격 + thco_sbst_pric: string; // 당사대용가격 + thco_sbst_pric_chng_dt: string; // 당사대용가격변경일자 + tr_stop_yn: string; // 거래정지여부 + admn_item_yn: string; // 관리종목여부 + thdt_clpr: string; // 당일종가 + bfdy_clpr: string; // 전일종가 + clpr_chng_dt: string; // 종가변경일자 + std_idst_clsf_cd: string; // 표준산업분류코드 + std_idst_clsf_cd_name: string; // 표준산업분류코드명 + idx_bztp_lcls_cd_name: string; // 지수업종대분류코드명 + idx_bztp_mcls_cd_name: string; // 지수업종중분류코드명 + idx_bztp_scls_cd_name: string; // 지수업종소분류코드명 + ocr_no: string; // OCR번호 + crfd_item_yn: string; // 크라우드펀딩종목여부 + elec_scty_yn: string; // 전자증권여부 + issu_istt_cd: string; // 발행기관코드 + etf_chas_erng_rt_dbnb: string; // ETF추적수익율배수 + etf_etn_ivst_heed_item_yn: string; // ETFETN투자유의종목여부 + stln_int_rt_dvsn_cd: string; // 대주이자율구분코드 + frnr_psnl_lmt_rt: string; // 외국인개인한도비율 + lstg_rqsr_issu_istt_cd: string; // 상장신청인발행기관코드 + lstg_rqsr_item_cd: string; // 상장신청인종목코드 + trst_istt_issu_istt_cd: string; // 신탁기관발행기관코드 +}; + +export const isProductDetail = (data: any): data is ProductDetail => { + return ( + typeof data.pdno === 'string' && + typeof data.prdt_type_cd === 'string' && + typeof data.mket_id_cd === 'string' && + typeof data.scty_grp_id_cd === 'string' && + typeof data.excg_dvsn_cd === 'string' && + typeof data.setl_mmdd === 'string' && + typeof data.lstg_stqt === 'string' && + typeof data.lstg_cptl_amt === 'string' && + typeof data.cpta === 'string' && + typeof data.papr === 'string' && + typeof data.issu_pric === 'string' && + typeof data.kospi200_item_yn === 'string' && + typeof data.scts_mket_lstg_dt === 'string' && + typeof data.scts_mket_lstg_abol_dt === 'string' && + typeof data.kosdaq_mket_lstg_dt === 'string' && + typeof data.kosdaq_mket_lstg_abol_dt === 'string' && + typeof data.frbd_mket_lstg_dt === 'string' && + typeof data.frbd_mket_lstg_abol_dt === 'string' && + typeof data.reits_kind_cd === 'string' && + typeof data.etf_dvsn_cd === 'string' && + typeof data.oilf_fund_yn === 'string' && + typeof data.idx_bztp_lcls_cd === 'string' && + typeof data.idx_bztp_mcls_cd === 'string' && + typeof data.idx_bztp_scls_cd === 'string' && + typeof data.stck_kind_cd === 'string' && + typeof data.mfnd_opng_dt === 'string' && + typeof data.mfnd_end_dt === 'string' && + typeof data.dpsi_erlm_cncl_dt === 'string' && + typeof data.etf_cu_qty === 'string' && + typeof data.prdt_name === 'string' && + typeof data.prdt_name120 === 'string' && + typeof data.prdt_abrv_name === 'string' && + typeof data.std_pdno === 'string' && + typeof data.prdt_eng_name === 'string' && + typeof data.prdt_eng_name120 === 'string' && + typeof data.prdt_eng_abrv_name === 'string' && + typeof data.dpsi_aptm_erlm_yn === 'string' && + typeof data.etf_txtn_type_cd === 'string' && + typeof data.etf_type_cd === 'string' && + typeof data.lstg_abol_dt === 'string' && + typeof data.nwst_odst_dvsn_cd === 'string' && + typeof data.sbst_pric === 'string' && + typeof data.thco_sbst_pric === 'string' && + typeof data.thco_sbst_pric_chng_dt === 'string' && + typeof data.tr_stop_yn === 'string' && + typeof data.admn_item_yn === 'string' && + typeof data.thdt_clpr === 'string' && + typeof data.bfdy_clpr === 'string' && + typeof data.clpr_chng_dt === 'string' && + typeof data.std_idst_clsf_cd === 'string' && + typeof data.std_idst_clsf_cd_name === 'string' && + typeof data.idx_bztp_lcls_cd_name === 'string' && + typeof data.idx_bztp_mcls_cd_name === 'string' && + typeof data.idx_bztp_scls_cd_name === 'string' && + typeof data.ocr_no === 'string' && + typeof data.crfd_item_yn === 'string' && + typeof data.elec_scty_yn === 'string' && + typeof data.issu_istt_cd === 'string' && + typeof data.etf_chas_erng_rt_dbnb === 'string' && + typeof data.etf_etn_ivst_heed_item_yn === 'string' && + typeof data.stln_int_rt_dvsn_cd === 'string' && + typeof data.frnr_psnl_lmt_rt === 'string' && + typeof data.lstg_rqsr_issu_istt_cd === 'string' && + typeof data.lstg_rqsr_item_cd === 'string' && + typeof data.trst_istt_issu_istt_cd === 'string' + ); +}; + +export type StockDetailQuery = { + pdno: string; + prdt_type_cd: string; +}; + +//export type FinancialDetail = { +// stac_yymm: string; // 결산 년월 +// sale_account: string; // 매출액 +// sale_cost: string; // 매출원가 +// sale_totl_prfi: string; // 매출총이익 +// depr_cost: string; // 감가상각비 +// sell_mang: string; // 판매관리비 +// bsop_prti: string; // 영업이익 +// bsop_non_ernn: string; // 영업외수익 +// bsop_non_expn: string; // 영업외비용 +// op_prfi: string; // 영업이익 +// spec_prfi: string; // 특별이익 +// spec_loss: string; // 특별손실 +// thtr_ntin: string; // 세전순이익 +//}; + +//export const isFinancialDetail = (data: any): data is FinancialDetail => { +// return ( +// typeof data.stac_yymm === 'string' && +// typeof data.sale_account === 'string' && +// typeof data.sale_cost === 'string' && +// typeof data.sale_totl_prfi === 'string' && +// typeof data.depr_cost === 'string' && +// typeof data.sell_mang === 'string' && +// typeof data.bsop_prti === 'string' && +// typeof data.bsop_non_ernn === 'string' && +// typeof data.bsop_non_expn === 'string' && +// typeof data.op_prfi === 'string' && +// typeof data.spec_prfi === 'string' && +// typeof data.spec_loss === 'string' && +// typeof data.thtr_ntin === 'string' +// ); +//}; diff --git a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts new file mode 100644 index 00000000..e1687cee --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable max-lines-per-function */ + +export type StockData = { + MKSC_SHRN_ISCD: string; // 유가증권 단축 종목코드 + STCK_CNTG_HOUR: string; // 주식 체결 시간 + STCK_PRPR: string; // 주식 현재가 + PRDY_VRSS_SIGN: string; // 전일 대비 부호 + PRDY_VRSS: string; // 전일 대비 + PRDY_CTRT: string; // 전일 대비율 + WGHN_AVRG_STCK_PRC: string; // 가중 평균 주식 가격 + STCK_OPRC: string; // 주식 시가 + STCK_HGPR: string; // 주식 최고가 + STCK_LWPR: string; // 주식 최저가 + ASKP1: string; // 매도호가1 + BIDP1: string; // 매수호가1 + CNTG_VOL: string; // 체결 거래량 + ACML_VOL: string; // 누적 거래량 + ACML_TR_PBMN: string; // 누적 거래 대금 + SELN_CNTG_CSNU: string; // 매도 체결 건수 + SHNU_CNTG_CSNU: string; // 매수 체결 건수 + NTBY_CNTG_CSNU: string; // 순매수 체결 건수 + CTTR: string; // 체결강도 + SELN_CNTG_SMTN: string; // 총 매도 수량 + SHNU_CNTG_SMTN: string; // 총 매수 수량 + CCLD_DVSN: string; // 체결구분 + SHNU_RATE: string; // 매수비율 + PRDY_VOL_VRSS_ACML_VOL_RATE: string; // 전일 거래량 대비 등락율 + OPRC_HOUR: string; // 시가 시간 + OPRC_VRSS_PRPR_SIGN: string; // 시가대비구분 + OPRC_VRSS_PRPR: string; // 시가대비 + HGPR_HOUR: string; // 최고가 시간 + HGPR_VRSS_PRPR_SIGN: string; // 고가대비구분 + HGPR_VRSS_PRPR: string; // 고가대비 + LWPR_HOUR: string; // 최저가 시간 + LWPR_VRSS_PRPR_SIGN: string; // 저가대비구분 + LWPR_VRSS_PRPR: string; // 저가대비 + BSOP_DATE: string; // 영업 일자 + NEW_MKOP_CLS_CODE: string; // 신 장운영 구분 코드 + TRHT_YN: string; // 거래정지 여부 + ASKP_RSQN1: string; // 매도호가 잔량1 + BIDP_RSQN1: string; // 매수호가 잔량1 + TOTAL_ASKP_RSQN: string; // 총 매도호가 잔량 + TOTAL_BIDP_RSQN: string; // 총 매수호가 잔량 + VOL_TNRT: string; // 거래량 회전율 + PRDY_SMNS_HOUR_ACML_VOL: string; // 전일 동시간 누적 거래량 + PRDY_SMNS_HOUR_ACML_VOL_RATE: string; // 전일 동시간 누적 거래량 비율 + HOUR_CLS_CODE: string; // 시간 구분 코드 + MRKT_TRTM_CLS_CODE: string; // 임의종료구분코드 + VI_STND_PRC: string; // 정적VI발동기준가 +}; + +export type OpenApiMessage = { + header: { + approval_key: string; + custtype: string; + tr_type: string; + 'content-type': string; + }; + body: { + input: { + tr_id: string; + tr_key: string; + }; + }; +}; + +export type MessageResponse = { + header: { + tr_id: string; + tr_key: string; + encrypt: string; + }; + body: { + rt_cd: string; + msg_cd: string; + msg1: string; + output?: { + iv: string; + key: string; + }; + }; +}; + +export function isMessageResponse(data: any): data is MessageResponse { + return ( + typeof data === 'object' && + data !== null && + typeof data.header === 'object' && + data.header !== null && + typeof data.header.tr_id === 'object' && + typeof data.header.tr_key === 'object' && + typeof data.header.encrypt === 'object' && + typeof data.body === 'object' && + data.body !== null && + typeof data.body.rt_cd === 'object' && + typeof data.body.msg_cd === 'object' && + typeof data.body.msg1 === 'object' && + typeof data.body.output === 'object' + ); +} + +export const stockDataKeys = [ + 'MKSC_SHRN_ISCD', + 'STCK_CNTG_HOUR', + 'STCK_PRPR', + 'PRDY_VRSS_SIGN', + 'PRDY_VRSS', + 'PRDY_CTRT', + 'WGHN_AVRG_STCK_PRC', + 'STCK_OPRC', + 'STCK_HGPR', + 'STCK_LWPR', + 'ASKP1', + 'BIDP1', + 'CNTG_VOL', + 'ACML_VOL', + 'ACML_TR_PBMN', + 'SELN_CNTG_CSNU', + 'SHNU_CNTG_CSNU', + 'NTBY_CNTG_CSNU', + 'CTTR', + 'SELN_CNTG_SMTN', + 'SHNU_CNTG_SMTN', + 'CCLD_DVSN', + 'SHNU_RATE', + 'PRDY_VOL_VRSS_ACML_VOL_RATE', + 'OPRC_HOUR', + 'OPRC_VRSS_PRPR_SIGN', + 'OPRC_VRSS_PRPR', + 'HGPR_HOUR', + 'HGPR_VRSS_PRPR_SIGN', + 'HGPR_VRSS_PRPR', + 'LWPR_HOUR', + 'LWPR_VRSS_PRPR_SIGN', + 'LWPR_VRSS_PRPR', + 'BSOP_DATE', + 'NEW_MKOP_CLS_CODE', + 'TRHT_YN', + 'ASKP_RSQN1', + 'BIDP_RSQN1', + 'TOTAL_ASKP_RSQN', + 'TOTAL_BIDP_RSQN', + 'VOL_TNRT', + 'PRDY_SMNS_HOUR_ACML_VOL', + 'PRDY_SMNS_HOUR_ACML_VOL_RATE', + 'HOUR_CLS_CODE', + 'MRKT_TRTM_CLS_CODE', + 'VI_STND_PRC', +]; diff --git a/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts b/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts new file mode 100644 index 00000000..5deb2d9e --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiMinuteData.type.ts @@ -0,0 +1,33 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +export type MinuteData = { + stck_bsop_date: string; + stck_cntg_hour: string; + stck_prpr: string; + stck_oprc: string; + stck_hgpr: string; + stck_lwpr: string; + cntg_vol: string; + acml_tr_pbmn: string; +}; + +export type UpdateStockQuery = { + fid_etc_cls_code: string; + fid_cond_mrkt_div_code: 'J' | 'W'; + fid_input_iscd: string; + fid_input_hour_1: string; + fid_pw_data_incu_yn: 'Y' | 'N'; +}; + +export const isMinuteData = (data: any) => { + return ( + typeof data.stck_bsop_date === 'string' && + typeof data.stck_cntg_hour === 'string' && + typeof data.stck_prpr === 'string' && + typeof data.stck_oprc === 'string' && + typeof data.stck_hgpr === 'string' && + typeof data.stck_lwpr === 'string' && + typeof data.cntg_vol === 'string' && + typeof data.acml_tr_pbmn === 'string' + ); +}; diff --git a/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts b/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts new file mode 100644 index 00000000..e4066f7c --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiPeriodData.ts @@ -0,0 +1,45 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +export type Period = 'D' | 'W' | 'M' | 'Y'; +export type ChartData = { + stck_bsop_date: string; + stck_clpr: string; + stck_oprc: string; + stck_hgpr: string; + stck_lwpr: string; + acml_vol: string; + acml_tr_pbmn: string; + flng_cls_code: string; + prtt_rate: string; + mod_yn: string; + prdy_vrss_sign: string; + prdy_vrss: string; + revl_issu_reas: string; +}; + +export type ItemChartPriceQuery = { + fid_cond_mrkt_div_code: 'J' | 'W'; + fid_input_iscd: string; + fid_input_date_1: string; + fid_input_date_2: string; + fid_period_div_code: Period; + fid_org_adj_prc: number; +}; + +export const isChartData = (data?: any) => { + return ( + data && + typeof data.stck_bsop_date === 'string' && + typeof data.stck_clpr === 'string' && + typeof data.stck_oprc === 'string' && + typeof data.stck_hgpr === 'string' && + typeof data.stck_lwpr === 'string' && + typeof data.acml_vol === 'string' && + typeof data.acml_tr_pbmn === 'string' && + typeof data.flng_cls_code === 'string' && + typeof data.prtt_rate === 'string' && + typeof data.mod_yn === 'string' && + typeof data.prdy_vrss_sign === 'string' && + typeof data.prdy_vrss === 'string' && + typeof data.revl_issu_reas === 'string' + ); +}; diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts new file mode 100644 index 00000000..6df0ca19 --- /dev/null +++ b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts @@ -0,0 +1,13 @@ +export type TR_ID = + | 'FHKST03010100' + | 'FHKST03010200' + | 'FHKST66430300' + | 'HHKDB669107C0' + | 'CTPF1002R'; + +export const TR_IDS: Record = { + ITEM_CHART_PRICE: 'FHKST03010100', + MINUTE_DATA: 'FHKST03010200', + FINANCIAL_DATA: 'FHKST66430300', + PRODUCTION_DETAIL: 'CTPF1002R', +}; diff --git a/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts b/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts new file mode 100644 index 00000000..1e0c3913 --- /dev/null +++ b/packages/backend/src/scraper/openapi/util/openapiCustom.error.ts @@ -0,0 +1,13 @@ +import { HttpException, HttpStatus } from '@nestjs/common'; + +export class OpenapiException extends HttpException { + private error: unknown; + constructor(message: string, status: HttpStatus, error?: unknown) { + super(message, status); + this.error = error; + } + + public getError() { + return this.error; + } +} diff --git a/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts new file mode 100644 index 00000000..fa8f75b4 --- /dev/null +++ b/packages/backend/src/scraper/openapi/util/openapiUtil.api.ts @@ -0,0 +1,115 @@ +/* eslint-disable @typescript-eslint/no-explicit-any*/ +import * as crypto from 'crypto'; +import { HttpStatus } from '@nestjs/common'; +import axios from 'axios'; +import { openApiConfig } from '../config/openapi.config'; +import { TR_ID } from '../type/openapiUtil.type'; +import { OpenapiException } from './openapiCustom.error'; + +const throwOpenapiException = (error: any) => { + if (error.message && error.response && error.response.status) { + throw new OpenapiException( + `Request failed: ${error.message}`, + error.response.status, + error, + ); + } else { + throw new OpenapiException( + `Unknown error: ${error.message || 'No message'}`, + HttpStatus.INTERNAL_SERVER_ERROR, + error, + ); + } +}; + +const postOpenApi = async ( + url: string, + config: typeof openApiConfig, + body: object, +) => { + try { + const response = await axios.post(config.STOCK_URL + url, body); + return response.data; + } catch (error) { + throwOpenapiException(error); + } +}; + +const getOpenApi = async ( + url: string, + config: typeof openApiConfig, + query: object, + tr_id: TR_ID, +) => { + try { + const response = await axios.get(config.STOCK_URL + url, { + params: query, + headers: { + Authorization: `Bearer ${config.STOCK_API_TOKEN}`, + appkey: config.STOCK_API_KEY, + appsecret: config.STOCK_API_PASSWORD, + tr_id, + custtype: 'P', + }, + }); + return response.data; + } catch (error) { + throwOpenapiException(error); + } +}; + +const getTodayDate = (): string => { + const today = new Date(); + return today.toISOString().split('T')[0].replace(/-/g, ''); +}; + +const getPreviousDate = (date: string, months: number): string => { + const currentDate = new Date( + date.slice(0, 4) + '-' + date.slice(4, 6) + '-' + date.slice(6, 8), + ); + currentDate.setMonth(currentDate.getMonth() - months); + return currentDate.toISOString().split('T')[0].replace(/-/g, ''); +}; + +const getCurrentTime = () => { + const now = new Date(); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + return `${hours}${minutes}${seconds}`; +}; + +const decryptAES256 = ( + encryptedText: string, + key: string, + iv: string, +): string => { + const decipher = crypto.createDecipheriv( + 'aes-256-cbc', + Buffer.from(key, 'hex'), + Buffer.from(iv, 'hex'), + ); + let decrypted = decipher.update(encryptedText, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + return decrypted; +}; + +const bufferToObject = (buffer: Buffer): any => { + try { + const jsonString = buffer.toString('utf-8'); + return JSON.parse(jsonString); + } catch (error) { + console.error('Failed to convert buffer to object:', error); + throw error; + } +}; + +export { + postOpenApi, + getOpenApi, + getTodayDate, + getPreviousDate, + getCurrentTime, + decryptAES256, + bufferToObject, +}; diff --git a/packages/backend/src/scraper/openapi/util/priorityQueue.ts b/packages/backend/src/scraper/openapi/util/priorityQueue.ts new file mode 100644 index 00000000..a49e5822 --- /dev/null +++ b/packages/backend/src/scraper/openapi/util/priorityQueue.ts @@ -0,0 +1,99 @@ +export class PriorityQueue { + private heap: { value: T; priority: number }[]; + + constructor() { + this.heap = []; + } + + private getParentIndex(index: number): number { + return Math.floor((index - 1) / 2); + } + + private getLeftChildIndex(index: number): number { + return index * 2 + 1; + } + + private getRightChildIndex(index: number): number { + return index * 2 + 2; + } + + private swap(i: number, j: number) { + [this.heap[i], this.heap[j]] = [this.heap[j], this.heap[i]]; + } + + private heapifyUp() { + let index = this.heap.length - 1; + while ( + index > 0 && + this.heap[index].priority < this.heap[this.getParentIndex(index)].priority + ) { + this.swap(index, this.getParentIndex(index)); + index = this.getParentIndex(index); + } + } + + private heapifyDown() { + let index = 0; + while (this.getLeftChildIndex(index) < this.heap.length) { + let smallerChildIndex = this.getLeftChildIndex(index); + const rightChildIndex = this.getRightChildIndex(index); + + if ( + rightChildIndex < this.heap.length && + this.heap[rightChildIndex].priority < + this.heap[smallerChildIndex].priority + ) { + smallerChildIndex = rightChildIndex; + } + + if (this.heap[index].priority <= this.heap[smallerChildIndex].priority) { + break; + } + + this.swap(index, smallerChildIndex); + index = smallerChildIndex; + } + } + + enqueue(value: T, priority: number) { + this.heap.push({ value, priority }); + this.heapifyUp(); + } + + dequeue(): T | undefined { + if (this.isEmpty()) { + return undefined; + } + + const root = this.heap[0]; + const last = this.heap.pop(); + + if (this.heap.length > 0 && last) { + this.heap[0] = last; + this.heapifyDown(); + } + + return root.value; + } + + peek(): T | undefined { + return this.heap.length > 0 ? this.heap[0].value : undefined; + } + + isEmpty(): boolean { + return this.heap.length === 0; + } +} + +const pq = new PriorityQueue(); + +pq.enqueue('Task A', 2); +pq.enqueue('Task B', 1); +pq.enqueue('Task C', 3); + +console.log(pq.dequeue()); // Task B +console.log(pq.peek()); // Task A +console.log(pq.dequeue()); // Task A +console.log(pq.isEmpty()); // false +console.log(pq.dequeue()); // Task C +console.log(pq.isEmpty()); // true diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts new file mode 100644 index 00000000..2e36acfb --- /dev/null +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -0,0 +1,148 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Inject, Injectable } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { Logger } from 'winston'; +import { RawData, WebSocket } from 'ws'; +import { OpenapiLiveData } from './api/openapiLiveData.api'; +import { openApiToken } from './api/openapiToken.api'; +import { openApiConfig } from './config/openapi.config'; +import { parseMessage } from './parse/openapi.parser'; + +type TR_IDS = '0' | '1'; + +@Injectable() +export class WebsocketClient { + private client: WebSocket; + private readonly reconnectInterval = 60000; + private readonly url = + process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; + private readonly clientStock: Set = new Set(); + + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly openapiLiveData: OpenapiLiveData, + ) { + if (process.env.NODE_ENV === 'production') { + this.connect(); + } + } + + // TODO : subscribe 구조로 리팩토링 + subscribe(stockId: string) { + this.clientStock.add(stockId); + // TODO : 하나의 config만 사용중. + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '1', + ); + this.sendMessage(message); + } + + discribe(stockId: string) { + this.clientStock.delete(stockId); + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '0', + ); + this.sendMessage(message); + } + + private initDisconnect() { + this.client.on('close', () => { + this.logger.warn( + `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, + ); + }); + + this.client.on('error', (error: any) => { + this.logger.error(`WebSocket error: ${error.message}`); + setTimeout(() => this.connect(), this.reconnectInterval); + }); + } + + private initOpen() { + this.client.on('open', () => { + this.logger.info('WebSocket connection established'); + for (const stockId of this.clientStock.keys()) { + const message = this.convertObjectToMessage( + openApiToken.configs[0], + stockId, + '1', + ); + this.sendMessage(message); + } + }); + } + + private initMessage() { + this.client.on('message', async (data) => { + try { + const message = this.parseMessage(data); + if (message.header) { + if (message.header.tr_id === 'PINGPONG') { + this.logger.info(`Received PING: ${JSON.stringify(data)}`); + this.client.pong({ + tr_id: 'PINGPONG', + datetime: new Date().toISOString(), + }); + } + return; + } + this.logger.info(`Recived data : ${data}`); + const liveData = this.openapiLiveData.convertLiveData(message); + this.openapiLiveData.saveLiveData(liveData); + } catch (error) { + this.logger.warn(error); + } + }); + } + + private parseMessage(data: RawData) { + if (typeof data === 'object') { + return data; + } else { + return parseMessage(data as string); + } + } + + @Cron('0 2 * * 1-5') + connect() { + this.client = new WebSocket(this.url); + this.initOpen(); + this.initMessage(); + this.initDisconnect(); + } + + private convertObjectToMessage( + config: typeof openApiConfig, + stockId: string, + tr_type: TR_IDS, + ): string { + const message = { + header: { + approval_key: config.STOCK_WEBSOCKET_KEY!, + custtype: 'P', + tr_type, + 'content-type': 'utf-8', + }, + body: { + input: { + tr_id: 'H0STCNT0', + tr_key: stockId, + }, + }, + }; + return JSON.stringify(message); + } + + private sendMessage(message: string) { + if (this.client.readyState === WebSocket.OPEN) { + this.client.send(message); + this.logger.info(`Sent message: ${message}`); + } else { + this.logger.warn('WebSocket is not open. Message not sent.'); + } + } +} From f612b6ea4c41950982abb65b5d903da7c3625693 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 22:22:15 +0900 Subject: [PATCH 085/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20websocket=20?= =?UTF-8?q?=EC=97=90=EB=9F=AC=20=ED=95=B4=EA=B2=B0,=20import,=20DI=20?= =?UTF-8?q?=EB=AC=B8=EC=A0=9C=20=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 279 ------------------ .../openapi/api/openapiLiveData.api.ts | 36 --- .../openapi/api/openapiMinuteData.api.ts | 156 ---------- .../openapi/api/openapiPeriodData.api.ts | 218 -------------- .../openapi/api/openapiToken.api.ts | 112 ------- .../openapi/config/openapi.config.ts | 17 -- .../openapi/openapi-scraper.module.ts | 43 --- .../openapi/openapi-scraper.service.ts | 15 - .../openapi/parse/openapi.parser.spec.ts | 106 ------- .../openapi/parse/openapi.parser.ts | 38 --- .../openapi/type/openapiDetailData.type.ts | 214 -------------- .../openapi/type/openapiLiveData.type.ts | 150 ---------- .../openapi/type/openapiMinuteData.type.ts | 33 --- .../openapi/type/openapiPeriodData.ts | 45 --- .../openapi/type/openapiUtil.type.ts | 13 - .../openapi/util/openapiCustom.error.ts | 13 - .../openapi/util/openapiUtil.api.ts | 115 -------- .../openapi/util/priorityQueue.ts | 99 ------- .../openapi/websocketClient.service.ts | 148 ---------- .../openapi/api/openapiLiveData.api.ts | 2 + .../scraper/openapi/openapi-scraper.module.ts | 1 + .../openapi/websocketClient.service.ts | 6 +- 22 files changed, 6 insertions(+), 1853 deletions(-) delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts delete mode 100644 packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts deleted file mode 100644 index 9be8e3ea..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiDetailData.api.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { Inject, Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Between, DataSource } from 'typeorm'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { - DetailDataQuery, - FinancialRatio, - isFinancialRatioData, - isProductDetail, - ProductDetail, - StockDetailQuery, -} from '../type/openapiDetailData.type'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { KospiStock } from '@/stock/domain/kospiStock.entity'; -import { Stock } from '@/stock/domain/stock.entity'; -import { StockDaily } from '@/stock/domain/stockData.entity'; -import { StockDetail } from '@/stock/domain/stockDetail.entity'; - -@Injectable() -export class OpenapiDetailData { - private readonly financialUrl: string = - '/uapi/domestic-stock/v1/finance/financial-ratio'; - private readonly productUrl: string = - '/uapi/domestic-stock/v1/quotations/search-stock-info'; - private readonly intervals = 1000; - constructor( - private readonly datasource: DataSource, - @Inject('winston') private readonly logger: Logger, - ) { - //setTimeout(() => this.getDetailData(), 5000); - } - - @Cron('0 8 * * 1-5') - async getDetailData() { - if (process.env.NODE_ENV !== 'production') return; - const entityManager = this.datasource.manager; - const stocks = await entityManager.find(Stock); - const configCount = openApiToken.configs.length; - const chunkSize = Math.ceil(stocks.length / configCount); - - for (let i = 0; i < configCount; i++) { - this.logger.info(openApiToken.configs[i]); - const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, openApiToken.configs[i]); - } - } - - private async saveDetailData(stockDetail: StockDetail) { - const manager = this.datasource.manager; - const entity = StockDetail; - const existingStockDetail = await manager.findOne(entity, { - where: { - stock: { id: stockDetail.stock.id }, - }, - }); - if (existingStockDetail) { - manager.update( - entity, - { stock: { id: stockDetail.stock.id } }, - stockDetail, - ); - } else { - manager.save(entity, stockDetail); - } - } - - private async saveKospiData(stockDetail: KospiStock) { - const manager = this.datasource.manager; - const entity = KospiStock; - const existingStockDetail = await manager.findOne(entity, { - where: { - stock: { id: stockDetail.stock.id }, - }, - }); - - if (existingStockDetail) { - manager.update( - entity, - { stock: { id: stockDetail.stock.id } }, - stockDetail, - ); - } else { - manager.save(entity, stockDetail); - } - } - - private async calPer(eps: number): Promise { - if (eps <= 0) return NaN; - const manager = this.datasource.manager; - const latestResult = await manager.find(StockDaily, { - skip: 0, - take: 1, - order: { createdAt: 'desc' }, - }); - // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 - if (latestResult && latestResult[0] && latestResult[0].close) { - const currentPrice = latestResult[0].close; - const per = currentPrice / eps; - - if (isNaN(per)) return 0; - else return per; - } else { - return 0; - } - } - - private async calMarketCap(lstg: number) { - const manager = this.datasource.manager; - const latestResult = await manager.find(StockDaily, { - skip: 0, - take: 1, - order: { createdAt: 'desc' }, - }); - - // TODO : price가 없는 경우 0으로 리턴, 나중에 NaN과 대응되게 리턴 - if (latestResult && latestResult[0] && latestResult[0].close) { - const currentPrice = latestResult[0].close; - const marketCap = lstg * currentPrice; - - if (isNaN(marketCap)) return 0; - else return marketCap; - } else { - return 0; - } - } - - private async get52WeeksLowHigh() { - const manager = this.datasource.manager; - const nowDate = new Date(); - const weeksAgoDate = this.getDate52WeeksAgo(); - // 주식의 52주간 일단위 데이터 전체 중에 최고, 최저가를 바탕으로 최저가, 최고가 계산해서 가져오기 - const output = await manager.find(StockDaily, { - select: ['low', 'high'], - where: { - startTime: Between(weeksAgoDate, nowDate), - }, - }); - const result = output.reduce((prev, cur) => { - if (prev.low > cur.low) prev.low = cur.low; - if (prev.high < cur.high) prev.high = cur.high; - return cur; - }, new StockDaily()); - let low = 0; - let high = 0; - if (result.low && !isNaN(result.low)) low = result.low; - if (result.high && !isNaN(result.high)) high = result.high; - return { low, high }; - } - - private async makeStockDetailObject( - output1: FinancialRatio, - output2: ProductDetail, - stockId: string, - ): Promise { - const result = new StockDetail(); - result.stock = { id: stockId } as Stock; - result.marketCap = - (await this.calMarketCap(parseInt(output2.lstg_stqt))) + ''; - result.eps = parseInt(output1.eps); - const { low, high } = await this.get52WeeksLowHigh(); - result.low52w = low; - result.high52w = high; - const eps = parseInt(output1.eps); - if (isNaN(eps)) result.eps = 0; - else result.eps = eps; - const per = await this.calPer(eps); - if (isNaN(per)) result.per = 0; - else result.per = per; - result.updatedAt = new Date(); - return result; - } - - private async makeKospiStockObject(output: ProductDetail, stockId: string) { - const ret = new KospiStock(); - ret.isKospi = output.kospi200_item_yn === 'Y' ? true : false; - ret.stock = { id: stockId } as Stock; - return ret; - } - - private async getFinancialRatio(stock: Stock, conf: typeof openApiConfig) { - const dataQuery = this.getDetailDataQuery(stock.id!); - // 여기서 가져올 건 eps -> eps와 per 계산하자. - try { - const response = await getOpenApi( - this.financialUrl, - conf, - dataQuery, - TR_IDS.FINANCIAL_DATA, - ); - if (response.output) { - const output1 = response.output; - return output1[0]; - } - } catch (error) { - this.logger.warn(error); - } - } - - private async getProductData(stock: Stock, conf: typeof openApiConfig) { - const defaultQuery = this.getFinancialDataQuery(stock.id!); - - // 여기서 가져올 건 lstg-stqt - 상장주수를 바탕으로 시가총액 계산, kospi200_item_yn 코스피200종목여부 업데이트 - try { - const response = await getOpenApi( - this.productUrl, - conf, - defaultQuery, - TR_IDS.PRODUCTION_DETAIL, - ); - if (response.output) { - const output2 = response.output; - return output2; - } - } catch (error) { - this.logger.warn(error); - } - } - - private async getDetailDataDelay(stock: Stock, conf: typeof openApiConfig) { - const output1 = await this.getFinancialRatio(stock, conf); - const output2 = await this.getProductData(stock, conf); - - this.logger.info(JSON.stringify(output1)); - this.logger.info(JSON.stringify(output2)); - if (isFinancialRatioData(output1) && isProductDetail(output2)) { - const stockDetail = await this.makeStockDetailObject( - output1, - output2, - stock.id!, - ); - this.saveDetailData(stockDetail); - const kospiStock = await this.makeKospiStockObject(output2, stock.id!); - this.saveKospiData(kospiStock); - - this.logger.info(`${stock.id!} is saved`); - } - } - - private async getDetailDataChunk(chunk: Stock[], conf: typeof openApiConfig) { - let delay = 0; - for await (const stock of chunk) { - setTimeout(() => this.getDetailDataDelay(stock, conf), delay); - delay += this.intervals; - } - } - - private getFinancialDataQuery( - stockId: string, - code: '300' | '301' | '302' | '306' = '300', - ): StockDetailQuery { - return { - pdno: stockId, - prdt_type_cd: code, - }; - } - - private getDetailDataQuery( - stockId: string, - divCode: 'J' = 'J', - classify: '0' | '1' = '0', - ): DetailDataQuery { - return { - fid_div_cls_code: classify, - fid_cond_mrkt_div_code: divCode, - fid_input_iscd: stockId, - }; - } - - private getDate52WeeksAgo(): Date { - const today = new Date(); - const weeksAgo = 52 * 7; - const date52WeeksAgo = new Date(today.setDate(today.getDate() - weeksAgo)); - date52WeeksAgo.setHours(0, 0, 0, 0); - return date52WeeksAgo; - } -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts deleted file mode 100644 index 410ab4cd..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiLiveData.api.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { DataSource } from 'typeorm'; -import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; - -export class OpenapiLiveData { - public readonly TR_ID: string = 'H0STCNT0'; - constructor( - private readonly datasource: DataSource, - ) {} - - async saveLiveData(data: StockLiveData[]) { - await this.datasource.manager - .getRepository(StockLiveData) - .createQueryBuilder() - .insert() - .into(StockLiveData) - .values(data) - .execute(); - } - - convertLiveData(messages: Record[]) : StockLiveData[] { - const stockData: StockLiveData[] = []; - messages.map((message) => { - const stockLiveData = new StockLiveData(); - stockLiveData.currentPrice = parseFloat(message.STCK_PRPR); - stockLiveData.changeRate = parseFloat(message.PRDY_CTRT); - stockLiveData.volume = parseInt(message.CNTG_VOL); - stockLiveData.high = parseFloat(message.STCK_HGPR); - stockLiveData.low = parseFloat(message.STCK_LWPR); - stockLiveData.open = parseFloat(message.STCK_OPRC); - stockLiveData.previousClose = parseFloat(message.WGHN_AVRG_STCK_PRC); - stockLiveData.updatedAt = new Date(); - stockData.push(stockLiveData); - }); - return stockData; - } -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts deleted file mode 100644 index 003e7560..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiMinuteData.api.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { Inject, Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { DataSource } from 'typeorm'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; - -import { - isMinuteData, - MinuteData, - UpdateStockQuery, -} from '../type/openapiMinuteData.type'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { Stock } from '@/stock/domain/stock.entity'; -import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; - -const STOCK_CUT = 4; - -@Injectable() -export class OpenapiMinuteData { - private stock: Stock[][] = []; - private readonly entity = StockMinutely; - private readonly url: string = - '/uapi/domestic-stock/v1/quotations/inquire-time-itemchartprice'; - private readonly intervals: number = 130; - private flip: number = 0; - constructor( - private readonly datasource: DataSource, - @Inject('winston') private readonly logger: Logger, - ) { - this.getStockData(); - } - - @Cron('0 1 * * 1-5') - async getStockData() { - if (process.env.NODE_ENV !== 'production') return; - const stock = await this.datasource.manager.findBy(Stock, { - isTrading: true, - }); - const stockSize = Math.ceil(stock.length / STOCK_CUT); - let i = 0; - this.stock = []; - while (i < STOCK_CUT) { - this.stock.push(stock.slice(i * stockSize, (i + 1) * stockSize)); - i++; - } - } - - private convertResToMinuteData( - stockId: string, - item: MinuteData, - time: string, - ) { - const stockPeriod = new StockData(); - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(item.stck_bsop_date.slice(0, 4)), - parseInt(item.stck_bsop_date.slice(4, 6)) - 1, - parseInt(item.stck_bsop_date.slice(6, 8)), - parseInt(time.slice(0, 2)), - parseInt(time.slice(2, 4)), - ); - stockPeriod.close = parseInt(item.stck_prpr); - stockPeriod.open = parseInt(item.stck_oprc); - stockPeriod.high = parseInt(item.stck_hgpr); - stockPeriod.low = parseInt(item.stck_lwpr); - stockPeriod.volume = parseInt(item.cntg_vol); - stockPeriod.createdAt = new Date(); - return stockPeriod; - } - - private isMarketOpenTime(time: string) { - const numberTime = parseInt(time); - return numberTime >= 90000 && numberTime <= 153000; - } - - private async saveMinuteData( - stockId: string, - item: MinuteData[], - time: string, - ) { - const manager = this.datasource.manager; - if (!this.isMarketOpenTime(time)) return; - const stockPeriod = item.map((val) => - this.convertResToMinuteData(stockId, val, time), - ); - manager.save(this.entity, stockPeriod); - } - - private async getMinuteDataInterval( - stockId: string, - time: string, - config: typeof openApiConfig, - ) { - const query = this.getUpdateStockQuery(stockId, time); - try { - const response = await getOpenApi( - this.url, - config, - query, - TR_IDS.MINUTE_DATA, - ); - let output; - if (response.output2) output = response.output2; - if (output && output[0] && isMinuteData(output[0])) { - this.saveMinuteData(stockId, output, time); - } - } catch (error) { - this.logger.warn(error); - } - } - - private async getMinuteDataChunk( - chunk: Stock[], - config: typeof openApiConfig, - ) { - const time = getCurrentTime(); - let interval = 0; - for await (const stock of chunk) { - setTimeout( - () => this.getMinuteDataInterval(stock.id!, time, config), - interval, - ); - interval += this.intervals; - } - } - - @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) - getMinuteData() { - if (process.env.NODE_ENV !== 'production') return; - const configCount = openApiToken.configs.length; - const stock = this.stock[this.flip % STOCK_CUT]; - this.flip++; - const chunkSize = Math.ceil(stock.length / configCount); - for (let i = 0; i < configCount; i++) { - const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); - this.getMinuteDataChunk(chunk, openApiToken.configs[i]); - } - } - - private getUpdateStockQuery( - stockId: string, - time: string, - isPastData: boolean = true, - marketCode: 'J' | 'W' = 'J', - ): UpdateStockQuery { - return { - fid_etc_cls_code: '', - fid_cond_mrkt_div_code: marketCode, - fid_input_iscd: stockId, - fid_input_hour_1: time, - fid_pw_data_incu_yn: isPastData ? 'Y' : 'N', - }; - } -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts deleted file mode 100644 index f4268088..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiPeriodData.api.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { Inject, Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { DataSource, EntityManager } from 'typeorm'; -import { Logger } from 'winston'; -import { - ChartData, - isChartData, - ItemChartPriceQuery, - Period, -} from '../type/openapiPeriodData'; -import { TR_IDS } from '../type/openapiUtil.type'; -import { - getOpenApi, - getPreviousDate, - getTodayDate, -} from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; -import { Stock } from '@/stock/domain/stock.entity'; -import { - StockData, - StockDaily, - StockWeekly, - StockMonthly, - StockYearly, -} from '@/stock/domain/stockData.entity'; - -const DATE_TO_ENTITY = { - D: StockDaily, - W: StockWeekly, - M: StockMonthly, - Y: StockYearly, -}; - -const DATE_TO_MONTH = { - D: 3, - W: 6, - M: 12, - Y: 24, -}; - -const INTERVALS = 4000; - -@Injectable() -export class OpenapiPeriodData { - private readonly url: string = - '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; - constructor( - private readonly datasource: DataSource, - @Inject('winston') private readonly logger: Logger, - ) { - //this.getItemChartPriceCheck(); - } - - @Cron('0 1 * * 1-5') - async getItemChartPriceCheck() { - if (process.env.NODE_ENV !== 'production') return; - const stocks = await this.datasource.manager.find(Stock, { - where: { - isTrading: true, - }, - }); - const configCount = openApiToken.configs.length; - const chunkSize = Math.ceil(stocks.length / configCount); - - for (let i = 0; i < configCount; i++) { - const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getChartData(chunk, 'D'); - setTimeout(() => this.getChartData(chunk, 'W'), INTERVALS); - setTimeout(() => this.getChartData(chunk, 'M'), INTERVALS * 2); - setTimeout(() => this.getChartData(chunk, 'Y'), INTERVALS * 3); - } - } - - private async getChartData(chunk: Stock[], period: Period) { - const baseTime = INTERVALS * 4; - const entity = DATE_TO_ENTITY[period]; - - let time = 0; - for (const stock of chunk) { - time += baseTime; - setTimeout(() => this.processStockData(stock, period, entity), time); - } - } - - private async processStockData( - stock: Stock, - period: Period, - entity: typeof StockData, - ) { - const stockPeriod = new StockData(); - const manager = this.datasource.manager; - let configIdx = 0; - let end = getTodayDate(); - let start = getPreviousDate(end, 3); - let isFail = false; - - while (!isFail) { - configIdx = (configIdx + 1) % openApiToken.configs.length; - this.setStockPeriod(stockPeriod, stock.id!, end); - - // chart 데이터가 있는 지 확인 -> 리턴 - if (await this.existsChartData(stockPeriod, manager, entity)) return; - - const query = this.getItemChartPriceQuery(stock.id!, start, end, period); - - const output = await this.fetchChartData(query, configIdx); - - if (output) { - await this.saveChartData(entity, stock.id!, output); - ({ endDate: end, startDate: start } = this.updateDates(start, period)); - } else isFail = true; - } - } - - private setStockPeriod( - stockPeriod: StockData, - stockId: string, - endDate: string, - ): void { - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(endDate.slice(0, 4)), - parseInt(endDate.slice(4, 6)) - 1, - parseInt(endDate.slice(6, 8)), - ); - } - - private async fetchChartData(query: ItemChartPriceQuery, configIdx: number) { - try { - const response = await getOpenApi( - this.url, - openApiToken.configs[configIdx], - query, - TR_IDS.ITEM_CHART_PRICE, - ); - return response.output2 as ChartData[]; - } catch (error) { - this.logger.warn(error); - } - } - - private updateDates( - startDate: string, - period: Period, - ): { endDate: string; startDate: string } { - const endDate = getPreviousDate(startDate, DATE_TO_MONTH[period]); - startDate = getPreviousDate(endDate, DATE_TO_MONTH[period]); - return { endDate, startDate }; - } - - private async existsChartData( - stock: StockData, - manager: EntityManager, - entity: typeof StockData, - ) { - return await manager.findOne(entity, { - where: { - stock: { id: stock.stock.id }, - createdAt: stock.startTime, - }, - }); - } - - private async insertChartData(stock: StockData, entity: typeof StockData) { - const manager = this.datasource.manager; - if (!(await this.existsChartData(stock, manager, entity))) { - await manager.save(entity, stock); - } - } - - private convertObjectToStockData(item: ChartData, stockId: string) { - const stockPeriod = new StockData(); - stockPeriod.stock = { id: stockId } as Stock; - stockPeriod.startTime = new Date( - parseInt(item.stck_bsop_date.slice(0, 4)), - parseInt(item.stck_bsop_date.slice(4, 6)) - 1, - parseInt(item.stck_bsop_date.slice(6, 8)), - ); - stockPeriod.close = parseInt(item.stck_clpr); - stockPeriod.open = parseInt(item.stck_oprc); - stockPeriod.high = parseInt(item.stck_hgpr); - stockPeriod.low = parseInt(item.stck_lwpr); - stockPeriod.volume = parseInt(item.acml_vol); - stockPeriod.createdAt = new Date(); - return stockPeriod; - } - - private async saveChartData( - entity: typeof StockData, - stockId: string, - data: ChartData[], - ) { - for (const item of data) { - if (!isChartData(item)) { - continue; - } - const stockPeriod = this.convertObjectToStockData(item, stockId); - await this.insertChartData(stockPeriod, entity); - } - } - - private getItemChartPriceQuery( - stockId: string, - startDate: string, - endDate: string, - period: Period, - marketCode: 'J' | 'W' = 'J', - ): ItemChartPriceQuery { - return { - fid_cond_mrkt_div_code: marketCode, - fid_input_iscd: stockId, - fid_input_date_1: startDate, - fid_input_date_2: endDate, - fid_period_div_code: period, - fid_org_adj_prc: 0, - }; - } -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts deleted file mode 100644 index 6e6c88c5..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/api/openapiToken.api.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { Inject } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Logger } from 'winston'; -import { openApiConfig } from '../config/openapi.config'; -import { OpenapiException } from '../util/openapiCustom.error'; -import { postOpenApi } from '../util/openapiUtil.api'; -import { logger } from '@/configs/logger.config'; - -class OpenapiTokenApi { - private config: (typeof openApiConfig)[] = []; - constructor(@Inject('winston') private readonly logger: Logger) { - const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); - const api_keys = openApiConfig.STOCK_API_KEY!.split(','); - const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); - if ( - accounts.length === 0 || - accounts.length !== api_keys.length || - api_passwords.length !== api_keys.length - ) { - this.logger.warn('Open API Config Error'); - } - for (let i = 0; i < accounts.length; i++) { - this.config.push({ - STOCK_URL: openApiConfig.STOCK_URL, - STOCK_ACCOUNT: accounts[i], - STOCK_API_KEY: api_keys[i], - STOCK_API_PASSWORD: api_passwords[i], - }); - } - this.initAuthenValue(); - } - - get configs() { - //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 - return this.config; - } - - private async initAuthenValue() { - const delay = 60000; - const delayMinute = delay / 1000 / 60; - - try { - await this.initAccessToken(); - await this.initWebSocketKey(); - } catch (error) { - if (error instanceof Error) { - this.logger.warn( - `Request failed: ${error.message}. Retrying in ${delayMinute} minute...`, - ); - } else { - this.logger.warn( - `Request failed. Retrying in ${delayMinute} minute...`, - ); - setTimeout(async () => { - await this.initAccessToken(); - await this.initWebSocketKey(); - }, delay); - } - } - } - - @Cron('50 0 * * 1-5') - async initAccessToken() { - const updatedConfig = await Promise.all( - this.config.map(async (val) => { - val.STOCK_API_TOKEN = await this.getToken(val)!; - return val; - }), - ); - this.config = updatedConfig; - } - - @Cron('50 0 * * 1-5') - async initWebSocketKey() { - const updatedConfig = await Promise.all( - this.config.map(async (val) => { - val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; - return val; - }), - ); - this.config = updatedConfig; - } - - private async getToken(config: typeof openApiConfig): Promise { - const body = { - grant_type: 'client_credentials', - appkey: config.STOCK_API_KEY, - appsecret: config.STOCK_API_PASSWORD, - }; - const tmp = await postOpenApi('/oauth2/tokenP', config, body); - if (!tmp.access_token) { - throw new OpenapiException('Access Token Failed', 403); - } - return tmp.access_token as string; - } - - private async getWebSocketKey(config: typeof openApiConfig): Promise { - const body = { - grant_type: 'client_credentials', - appkey: config.STOCK_API_KEY, - secretkey: config.STOCK_API_PASSWORD, - }; - const tmp = await postOpenApi('/oauth2/Approval', config, body); - if (!tmp.approval_key) { - throw new OpenapiException('WebSocket Key Failed', 403); - } - return tmp.approval_key as string; - } -} - -const openApiToken = new OpenapiTokenApi(logger); -export { openApiToken }; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts b/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts deleted file mode 100644 index 8aa12ea3..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/config/openapi.config.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as dotenv from 'dotenv'; - -dotenv.config(); - -export const openApiConfig: { - STOCK_URL: string | undefined; - STOCK_ACCOUNT: string | undefined; - STOCK_API_KEY: string | undefined; - STOCK_API_PASSWORD: string | undefined; - STOCK_API_TOKEN?: string; - STOCK_WEBSOCKET_KEY?: string; -} = { - STOCK_URL: process.env.STOCK_URL, - STOCK_ACCOUNT: process.env.STOCK_ACCOUNT, - STOCK_API_KEY: process.env.STOCK_API_KEY, - STOCK_API_PASSWORD: process.env.STOCK_API_PASSWORD, -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts deleted file mode 100644 index cb45c91c..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.module.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { OpenapiDetailData } from './api/openapiDetailData.api'; -import { OpenapiLiveData } from './api/openapiLiveData.api'; -import { OpenapiMinuteData } from './api/openapiMinuteData.api'; -import { OpenapiPeriodData } from './api/openapiPeriodData.api'; -import { OpenapiScraperService } from './openapi-scraper.service'; -import { WebsocketClient } from './websocketClient.service'; -import { Stock } from '@/stock/domain/stock.entity'; -import { - StockDaily, - StockMinutely, - StockMonthly, - StockWeekly, - StockYearly, -} from '@/stock/domain/stockData.entity'; -import { StockDetail } from '@/stock/domain/stockDetail.entity'; -import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; - -@Module({ - imports: [ - TypeOrmModule.forFeature([ - Stock, - StockMinutely, - StockDaily, - StockWeekly, - StockMonthly, - StockYearly, - StockLiveData, - StockDetail, - ]), - ], - controllers: [], - providers: [ - OpenapiPeriodData, - OpenapiMinuteData, - OpenapiDetailData, - OpenapiScraperService, - OpenapiLiveData, - WebsocketClient, - ], -}) -export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts deleted file mode 100644 index 52c90179..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/openapi-scraper.service.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; -import { OpenapiDetailData } from './api/openapiDetailData.api'; -import { OpenapiMinuteData } from './api/openapiMinuteData.api'; -import { OpenapiPeriodData } from './api/openapiPeriodData.api'; - -@Injectable() -export class OpenapiScraperService { - public constructor( - private datasource: DataSource, - private readonly openapiPeriodData: OpenapiPeriodData, - private readonly openapiMinuteData: OpenapiMinuteData, - private readonly openapiDetailData: OpenapiDetailData, - ) {} -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts deleted file mode 100644 index 37d3deb6..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.spec.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* eslint-disable max-lines-per-function */ -import { parseMessage } from './openapi.parser'; - -const answer = [ - { - STOCK_ID: '005930', - MKSC_SHRN_ISCD: 5930, - STCK_CNTG_HOUR: 93354, - STCK_PRPR: 71900, - PRDY_VRSS_SIGN: 5, - PRDY_VRSS: -100, - PRDY_CTRT: -0.14, - WGHN_AVRG_STCK_PRC: 72023.83, - STCK_OPRC: 72100, - STCK_HGPR: 72400, - STCK_LWPR: 71700, - ASKP1: 71900, - BIDP1: 71800, - CNTG_VOL: 1, - ACML_VOL: 3052507, - ACML_TR_PBMN: 219853241700, - SELN_CNTG_CSNU: 5105, - SHNU_CNTG_CSNU: 6937, - NTBY_CNTG_CSNU: 1832, - CTTR: 84.9, - SELN_CNTG_SMTN: 1366314, - SHNU_CNTG_SMTN: 1159996, - CCLD_DVSN: 1, - SHNU_RATE: 0.39, - PRDY_VOL_VRSS_ACML_VOL_RATE: 20.28, - OPRC_HOUR: 90020, - OPRC_VRSS_PRPR_SIGN: 5, - OPRC_VRSS_PRPR: -200, - HGPR_HOUR: 90820, - HGPR_VRSS_PRPR_SIGN: 5, - HGPR_VRSS_PRPR: -500, - LWPR_HOUR: 92619, - LWPR_VRSS_PRPR_SIGN: 2, - LWPR_VRSS_PRPR: 200, - BSOP_DATE: 20230612, - NEW_MKOP_CLS_CODE: 20, - TRHT_YN: 'N', - ASKP_RSQN1: 65945, - BIDP_RSQN1: 216924, - TOTAL_ASKP_RSQN: 1118750, - TOTAL_BIDP_RSQN: 2199206, - VOL_TNRT: 0.05, - PRDY_SMNS_HOUR_ACML_VOL: 2424142, - PRDY_SMNS_HOUR_ACML_VOL_RATE: 125.92, - HOUR_CLS_CODE: 0, - MRKT_TRTM_CLS_CODE: null, - VI_STND_PRC: 72100, - }, -]; - -describe('openapi parser test', () => { - test('parse json websocket data', () => { - const message = `{ - "header": { - "tr_id": "H0STCNT0", - "tr_key": "005930", - "encrypt": "N" - }, - "body": { - "rt_cd": "0", - "msg_cd": "OPSP0000", - "msg1": "SUBSCRIBE SUCCESS", - "output": { - "iv": "0123456789abcdef", - "key": "abcdefghijklmnopabcdefghijklmnop"} - } - }`; - - const result = parseMessage(message); - - expect(result).toEqual(JSON.parse(message)); - }); - - test('parse stockData', () => { - const message = - '0|H0STCNT0|001|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + - '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + - '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + - '2424142^125.92^0^^72100'; - - const result = parseMessage(message); - - expect(result).toEqual(answer); - }); - - test('parse stockData', () => { - const message = - '0|H0STCNT0|002|005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + - '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + - '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + - '2424142^125.92^0^^72100^' + - '005930^093354^71900^5^-100^-0.14^72023.83^72100^72400^71700^71900^71800^1^3052' + - '507^219853241700^5105^6937^1832^84.90^1366314^1159996^1^0.39^20.28^090020^5^-2' + - '00^090820^5^-500^092619^2^200^20230612^20^N^65945^216924^1118750^2199206^0.05^' + - '2424142^125.92^0^^72100'; - - const result = parseMessage(message); - - expect(result).toEqual([answer[0], answer[0]]); - }); -}); diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts b/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts deleted file mode 100644 index fe3e7002..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/parse/openapi.parser.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { stockDataKeys } from '../type/openapiLiveData.type'; - -export const parseMessage = (data: string) => { - try { - return JSON.parse(data); - //eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (e) { - return parseStockData(data); - } -}; -const FIELD_LENGTH: number = stockDataKeys.length; - -const parseStockData = (input: string) => { - const dataBlocks = input.split('|'); // 데이터 구분 - const results = []; - const size = parseInt(dataBlocks[2]); // 데이터 건수 - const rawData = dataBlocks[3]; - const values = rawData.split('^'); // 필드 구분자 '^' - - for (let i = 0; i < size; i++) { - //TODO : type narrowing require - const parsedData: Record = {}; - parsedData['STOCK_ID'] = values[i * FIELD_LENGTH]; - stockDataKeys.forEach((field: string, index: number) => { - const value = values[index + FIELD_LENGTH * i]; - if (!value) return (parsedData[field] = null); - - // 숫자형 필드 처리 - if (isNaN(parseInt(value))) { - parsedData[field] = value; // 문자열 그대로 저장 - } else { - parsedData[field] = parseFloat(value); // 숫자로 변환 - } - }); - results.push(parsedData); - } - return results; -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts deleted file mode 100644 index 38015d48..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiDetailData.type.ts +++ /dev/null @@ -1,214 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any*/ -/* eslint-disable max-lines-per-function */ - -export type DetailDataQuery = { - fid_cond_mrkt_div_code: 'J'; - fid_input_iscd: string; - fid_div_cls_code: '0' | '1'; -}; - -export type FinancialRatio = { - stac_yymm: string; // 결산 년월 - grs: string; // 매출액 증가율 - bsop_prfi_inrt: string; // 영업 이익 증가율 - ntin_inrt: string; // 순이익 증가율 - roe_val: string; // ROE 값 - eps: string; // EPS - sps: string; // 주당매출액 - bps: string; // BPS - rsrv_rate: string; // 유보 비율 - lblt_rate: string; // 부채 비율 -}; - -export function isFinancialRatioData(data: any): data is FinancialRatio { - return ( - data && - typeof data.stac_yymm === 'string' && - typeof data.grs === 'string' && - typeof data.bsop_prfi_inrt === 'string' && - typeof data.ntin_inrt === 'string' && - typeof data.roe_val === 'string' && - typeof data.eps === 'string' && - typeof data.sps === 'string' && - typeof data.bps === 'string' && - typeof data.rsrv_rate === 'string' && - typeof data.lblt_rate === 'string' - ); -} - -export type ProductDetail = { - pdno: string; // 상품번호 - prdt_type_cd: string; // 상품유형코드 - mket_id_cd: string; // 시장ID코드 - scty_grp_id_cd: string; // 증권그룹ID코드 - excg_dvsn_cd: string; // 거래소구분코드 - setl_mmdd: string; // 결산월일 - lstg_stqt: string; // 상장주수 - 이거 사용 - lstg_cptl_amt: string; // 상장자본금액 - cpta: string; // 자본금 - papr: string; // 액면가 - issu_pric: string; // 발행가격 - kospi200_item_yn: string; // 코스피200종목여부 - 이것도 사용 - scts_mket_lstg_dt: string; // 유가증권시장상장일자 - scts_mket_lstg_abol_dt: string; // 유가증권시장상장폐지일자 - kosdaq_mket_lstg_dt: string; // 코스닥시장상장일자 - kosdaq_mket_lstg_abol_dt: string; // 코스닥시장상장폐지일자 - frbd_mket_lstg_dt: string; // 프리보드시장상장일자 - frbd_mket_lstg_abol_dt: string; // 프리보드시장상장폐지일자 - reits_kind_cd: string; // 리츠종류코드 - etf_dvsn_cd: string; // ETF구분코드 - oilf_fund_yn: string; // 유전펀드여부 - idx_bztp_lcls_cd: string; // 지수업종대분류코드 - idx_bztp_mcls_cd: string; // 지수업종중분류코드 - idx_bztp_scls_cd: string; // 지수업종소분류코드 - stck_kind_cd: string; // 주식종류코드 - mfnd_opng_dt: string; // 뮤추얼펀드개시일자 - mfnd_end_dt: string; // 뮤추얼펀드종료일자 - dpsi_erlm_cncl_dt: string; // 예탁등록취소일자 - etf_cu_qty: string; // ETFCU수량 - prdt_name: string; // 상품명 - prdt_name120: string; // 상품명120 - prdt_abrv_name: string; // 상품약어명 - std_pdno: string; // 표준상품번호 - prdt_eng_name: string; // 상품영문명 - prdt_eng_name120: string; // 상품영문명120 - prdt_eng_abrv_name: string; // 상품영문약어명 - dpsi_aptm_erlm_yn: string; // 예탁지정등록여부 - etf_txtn_type_cd: string; // ETF과세유형코드 - etf_type_cd: string; // ETF유형코드 - lstg_abol_dt: string; // 상장폐지일자 - nwst_odst_dvsn_cd: string; // 신주구주구분코드 - sbst_pric: string; // 대용가격 - thco_sbst_pric: string; // 당사대용가격 - thco_sbst_pric_chng_dt: string; // 당사대용가격변경일자 - tr_stop_yn: string; // 거래정지여부 - admn_item_yn: string; // 관리종목여부 - thdt_clpr: string; // 당일종가 - bfdy_clpr: string; // 전일종가 - clpr_chng_dt: string; // 종가변경일자 - std_idst_clsf_cd: string; // 표준산업분류코드 - std_idst_clsf_cd_name: string; // 표준산업분류코드명 - idx_bztp_lcls_cd_name: string; // 지수업종대분류코드명 - idx_bztp_mcls_cd_name: string; // 지수업종중분류코드명 - idx_bztp_scls_cd_name: string; // 지수업종소분류코드명 - ocr_no: string; // OCR번호 - crfd_item_yn: string; // 크라우드펀딩종목여부 - elec_scty_yn: string; // 전자증권여부 - issu_istt_cd: string; // 발행기관코드 - etf_chas_erng_rt_dbnb: string; // ETF추적수익율배수 - etf_etn_ivst_heed_item_yn: string; // ETFETN투자유의종목여부 - stln_int_rt_dvsn_cd: string; // 대주이자율구분코드 - frnr_psnl_lmt_rt: string; // 외국인개인한도비율 - lstg_rqsr_issu_istt_cd: string; // 상장신청인발행기관코드 - lstg_rqsr_item_cd: string; // 상장신청인종목코드 - trst_istt_issu_istt_cd: string; // 신탁기관발행기관코드 -}; - -export const isProductDetail = (data: any): data is ProductDetail => { - return ( - typeof data.pdno === 'string' && - typeof data.prdt_type_cd === 'string' && - typeof data.mket_id_cd === 'string' && - typeof data.scty_grp_id_cd === 'string' && - typeof data.excg_dvsn_cd === 'string' && - typeof data.setl_mmdd === 'string' && - typeof data.lstg_stqt === 'string' && - typeof data.lstg_cptl_amt === 'string' && - typeof data.cpta === 'string' && - typeof data.papr === 'string' && - typeof data.issu_pric === 'string' && - typeof data.kospi200_item_yn === 'string' && - typeof data.scts_mket_lstg_dt === 'string' && - typeof data.scts_mket_lstg_abol_dt === 'string' && - typeof data.kosdaq_mket_lstg_dt === 'string' && - typeof data.kosdaq_mket_lstg_abol_dt === 'string' && - typeof data.frbd_mket_lstg_dt === 'string' && - typeof data.frbd_mket_lstg_abol_dt === 'string' && - typeof data.reits_kind_cd === 'string' && - typeof data.etf_dvsn_cd === 'string' && - typeof data.oilf_fund_yn === 'string' && - typeof data.idx_bztp_lcls_cd === 'string' && - typeof data.idx_bztp_mcls_cd === 'string' && - typeof data.idx_bztp_scls_cd === 'string' && - typeof data.stck_kind_cd === 'string' && - typeof data.mfnd_opng_dt === 'string' && - typeof data.mfnd_end_dt === 'string' && - typeof data.dpsi_erlm_cncl_dt === 'string' && - typeof data.etf_cu_qty === 'string' && - typeof data.prdt_name === 'string' && - typeof data.prdt_name120 === 'string' && - typeof data.prdt_abrv_name === 'string' && - typeof data.std_pdno === 'string' && - typeof data.prdt_eng_name === 'string' && - typeof data.prdt_eng_name120 === 'string' && - typeof data.prdt_eng_abrv_name === 'string' && - typeof data.dpsi_aptm_erlm_yn === 'string' && - typeof data.etf_txtn_type_cd === 'string' && - typeof data.etf_type_cd === 'string' && - typeof data.lstg_abol_dt === 'string' && - typeof data.nwst_odst_dvsn_cd === 'string' && - typeof data.sbst_pric === 'string' && - typeof data.thco_sbst_pric === 'string' && - typeof data.thco_sbst_pric_chng_dt === 'string' && - typeof data.tr_stop_yn === 'string' && - typeof data.admn_item_yn === 'string' && - typeof data.thdt_clpr === 'string' && - typeof data.bfdy_clpr === 'string' && - typeof data.clpr_chng_dt === 'string' && - typeof data.std_idst_clsf_cd === 'string' && - typeof data.std_idst_clsf_cd_name === 'string' && - typeof data.idx_bztp_lcls_cd_name === 'string' && - typeof data.idx_bztp_mcls_cd_name === 'string' && - typeof data.idx_bztp_scls_cd_name === 'string' && - typeof data.ocr_no === 'string' && - typeof data.crfd_item_yn === 'string' && - typeof data.elec_scty_yn === 'string' && - typeof data.issu_istt_cd === 'string' && - typeof data.etf_chas_erng_rt_dbnb === 'string' && - typeof data.etf_etn_ivst_heed_item_yn === 'string' && - typeof data.stln_int_rt_dvsn_cd === 'string' && - typeof data.frnr_psnl_lmt_rt === 'string' && - typeof data.lstg_rqsr_issu_istt_cd === 'string' && - typeof data.lstg_rqsr_item_cd === 'string' && - typeof data.trst_istt_issu_istt_cd === 'string' - ); -}; - -export type StockDetailQuery = { - pdno: string; - prdt_type_cd: string; -}; - -//export type FinancialDetail = { -// stac_yymm: string; // 결산 년월 -// sale_account: string; // 매출액 -// sale_cost: string; // 매출원가 -// sale_totl_prfi: string; // 매출총이익 -// depr_cost: string; // 감가상각비 -// sell_mang: string; // 판매관리비 -// bsop_prti: string; // 영업이익 -// bsop_non_ernn: string; // 영업외수익 -// bsop_non_expn: string; // 영업외비용 -// op_prfi: string; // 영업이익 -// spec_prfi: string; // 특별이익 -// spec_loss: string; // 특별손실 -// thtr_ntin: string; // 세전순이익 -//}; - -//export const isFinancialDetail = (data: any): data is FinancialDetail => { -// return ( -// typeof data.stac_yymm === 'string' && -// typeof data.sale_account === 'string' && -// typeof data.sale_cost === 'string' && -// typeof data.sale_totl_prfi === 'string' && -// typeof data.depr_cost === 'string' && -// typeof data.sell_mang === 'string' && -// typeof data.bsop_prti === 'string' && -// typeof data.bsop_non_ernn === 'string' && -// typeof data.bsop_non_expn === 'string' && -// typeof data.op_prfi === 'string' && -// typeof data.spec_prfi === 'string' && -// typeof data.spec_loss === 'string' && -// typeof data.thtr_ntin === 'string' -// ); -//}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts deleted file mode 100644 index e1687cee..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiLiveData.type.ts +++ /dev/null @@ -1,150 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable max-lines-per-function */ - -export type StockData = { - MKSC_SHRN_ISCD: string; // 유가증권 단축 종목코드 - STCK_CNTG_HOUR: string; // 주식 체결 시간 - STCK_PRPR: string; // 주식 현재가 - PRDY_VRSS_SIGN: string; // 전일 대비 부호 - PRDY_VRSS: string; // 전일 대비 - PRDY_CTRT: string; // 전일 대비율 - WGHN_AVRG_STCK_PRC: string; // 가중 평균 주식 가격 - STCK_OPRC: string; // 주식 시가 - STCK_HGPR: string; // 주식 최고가 - STCK_LWPR: string; // 주식 최저가 - ASKP1: string; // 매도호가1 - BIDP1: string; // 매수호가1 - CNTG_VOL: string; // 체결 거래량 - ACML_VOL: string; // 누적 거래량 - ACML_TR_PBMN: string; // 누적 거래 대금 - SELN_CNTG_CSNU: string; // 매도 체결 건수 - SHNU_CNTG_CSNU: string; // 매수 체결 건수 - NTBY_CNTG_CSNU: string; // 순매수 체결 건수 - CTTR: string; // 체결강도 - SELN_CNTG_SMTN: string; // 총 매도 수량 - SHNU_CNTG_SMTN: string; // 총 매수 수량 - CCLD_DVSN: string; // 체결구분 - SHNU_RATE: string; // 매수비율 - PRDY_VOL_VRSS_ACML_VOL_RATE: string; // 전일 거래량 대비 등락율 - OPRC_HOUR: string; // 시가 시간 - OPRC_VRSS_PRPR_SIGN: string; // 시가대비구분 - OPRC_VRSS_PRPR: string; // 시가대비 - HGPR_HOUR: string; // 최고가 시간 - HGPR_VRSS_PRPR_SIGN: string; // 고가대비구분 - HGPR_VRSS_PRPR: string; // 고가대비 - LWPR_HOUR: string; // 최저가 시간 - LWPR_VRSS_PRPR_SIGN: string; // 저가대비구분 - LWPR_VRSS_PRPR: string; // 저가대비 - BSOP_DATE: string; // 영업 일자 - NEW_MKOP_CLS_CODE: string; // 신 장운영 구분 코드 - TRHT_YN: string; // 거래정지 여부 - ASKP_RSQN1: string; // 매도호가 잔량1 - BIDP_RSQN1: string; // 매수호가 잔량1 - TOTAL_ASKP_RSQN: string; // 총 매도호가 잔량 - TOTAL_BIDP_RSQN: string; // 총 매수호가 잔량 - VOL_TNRT: string; // 거래량 회전율 - PRDY_SMNS_HOUR_ACML_VOL: string; // 전일 동시간 누적 거래량 - PRDY_SMNS_HOUR_ACML_VOL_RATE: string; // 전일 동시간 누적 거래량 비율 - HOUR_CLS_CODE: string; // 시간 구분 코드 - MRKT_TRTM_CLS_CODE: string; // 임의종료구분코드 - VI_STND_PRC: string; // 정적VI발동기준가 -}; - -export type OpenApiMessage = { - header: { - approval_key: string; - custtype: string; - tr_type: string; - 'content-type': string; - }; - body: { - input: { - tr_id: string; - tr_key: string; - }; - }; -}; - -export type MessageResponse = { - header: { - tr_id: string; - tr_key: string; - encrypt: string; - }; - body: { - rt_cd: string; - msg_cd: string; - msg1: string; - output?: { - iv: string; - key: string; - }; - }; -}; - -export function isMessageResponse(data: any): data is MessageResponse { - return ( - typeof data === 'object' && - data !== null && - typeof data.header === 'object' && - data.header !== null && - typeof data.header.tr_id === 'object' && - typeof data.header.tr_key === 'object' && - typeof data.header.encrypt === 'object' && - typeof data.body === 'object' && - data.body !== null && - typeof data.body.rt_cd === 'object' && - typeof data.body.msg_cd === 'object' && - typeof data.body.msg1 === 'object' && - typeof data.body.output === 'object' - ); -} - -export const stockDataKeys = [ - 'MKSC_SHRN_ISCD', - 'STCK_CNTG_HOUR', - 'STCK_PRPR', - 'PRDY_VRSS_SIGN', - 'PRDY_VRSS', - 'PRDY_CTRT', - 'WGHN_AVRG_STCK_PRC', - 'STCK_OPRC', - 'STCK_HGPR', - 'STCK_LWPR', - 'ASKP1', - 'BIDP1', - 'CNTG_VOL', - 'ACML_VOL', - 'ACML_TR_PBMN', - 'SELN_CNTG_CSNU', - 'SHNU_CNTG_CSNU', - 'NTBY_CNTG_CSNU', - 'CTTR', - 'SELN_CNTG_SMTN', - 'SHNU_CNTG_SMTN', - 'CCLD_DVSN', - 'SHNU_RATE', - 'PRDY_VOL_VRSS_ACML_VOL_RATE', - 'OPRC_HOUR', - 'OPRC_VRSS_PRPR_SIGN', - 'OPRC_VRSS_PRPR', - 'HGPR_HOUR', - 'HGPR_VRSS_PRPR_SIGN', - 'HGPR_VRSS_PRPR', - 'LWPR_HOUR', - 'LWPR_VRSS_PRPR_SIGN', - 'LWPR_VRSS_PRPR', - 'BSOP_DATE', - 'NEW_MKOP_CLS_CODE', - 'TRHT_YN', - 'ASKP_RSQN1', - 'BIDP_RSQN1', - 'TOTAL_ASKP_RSQN', - 'TOTAL_BIDP_RSQN', - 'VOL_TNRT', - 'PRDY_SMNS_HOUR_ACML_VOL', - 'PRDY_SMNS_HOUR_ACML_VOL_RATE', - 'HOUR_CLS_CODE', - 'MRKT_TRTM_CLS_CODE', - 'VI_STND_PRC', -]; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts deleted file mode 100644 index 5deb2d9e..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiMinuteData.type.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -export type MinuteData = { - stck_bsop_date: string; - stck_cntg_hour: string; - stck_prpr: string; - stck_oprc: string; - stck_hgpr: string; - stck_lwpr: string; - cntg_vol: string; - acml_tr_pbmn: string; -}; - -export type UpdateStockQuery = { - fid_etc_cls_code: string; - fid_cond_mrkt_div_code: 'J' | 'W'; - fid_input_iscd: string; - fid_input_hour_1: string; - fid_pw_data_incu_yn: 'Y' | 'N'; -}; - -export const isMinuteData = (data: any) => { - return ( - typeof data.stck_bsop_date === 'string' && - typeof data.stck_cntg_hour === 'string' && - typeof data.stck_prpr === 'string' && - typeof data.stck_oprc === 'string' && - typeof data.stck_hgpr === 'string' && - typeof data.stck_lwpr === 'string' && - typeof data.cntg_vol === 'string' && - typeof data.acml_tr_pbmn === 'string' - ); -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts deleted file mode 100644 index e4066f7c..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiPeriodData.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -export type Period = 'D' | 'W' | 'M' | 'Y'; -export type ChartData = { - stck_bsop_date: string; - stck_clpr: string; - stck_oprc: string; - stck_hgpr: string; - stck_lwpr: string; - acml_vol: string; - acml_tr_pbmn: string; - flng_cls_code: string; - prtt_rate: string; - mod_yn: string; - prdy_vrss_sign: string; - prdy_vrss: string; - revl_issu_reas: string; -}; - -export type ItemChartPriceQuery = { - fid_cond_mrkt_div_code: 'J' | 'W'; - fid_input_iscd: string; - fid_input_date_1: string; - fid_input_date_2: string; - fid_period_div_code: Period; - fid_org_adj_prc: number; -}; - -export const isChartData = (data?: any) => { - return ( - data && - typeof data.stck_bsop_date === 'string' && - typeof data.stck_clpr === 'string' && - typeof data.stck_oprc === 'string' && - typeof data.stck_hgpr === 'string' && - typeof data.stck_lwpr === 'string' && - typeof data.acml_vol === 'string' && - typeof data.acml_tr_pbmn === 'string' && - typeof data.flng_cls_code === 'string' && - typeof data.prtt_rate === 'string' && - typeof data.mod_yn === 'string' && - typeof data.prdy_vrss_sign === 'string' && - typeof data.prdy_vrss === 'string' && - typeof data.revl_issu_reas === 'string' - ); -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts deleted file mode 100644 index 6df0ca19..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/type/openapiUtil.type.ts +++ /dev/null @@ -1,13 +0,0 @@ -export type TR_ID = - | 'FHKST03010100' - | 'FHKST03010200' - | 'FHKST66430300' - | 'HHKDB669107C0' - | 'CTPF1002R'; - -export const TR_IDS: Record = { - ITEM_CHART_PRICE: 'FHKST03010100', - MINUTE_DATA: 'FHKST03010200', - FINANCIAL_DATA: 'FHKST66430300', - PRODUCTION_DETAIL: 'CTPF1002R', -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts deleted file mode 100644 index 1e0c3913..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiCustom.error.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; - -export class OpenapiException extends HttpException { - private error: unknown; - constructor(message: string, status: HttpStatus, error?: unknown) { - super(message, status); - this.error = error; - } - - public getError() { - return this.error; - } -} diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts deleted file mode 100644 index fa8f75b4..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/util/openapiUtil.api.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any*/ -import * as crypto from 'crypto'; -import { HttpStatus } from '@nestjs/common'; -import axios from 'axios'; -import { openApiConfig } from '../config/openapi.config'; -import { TR_ID } from '../type/openapiUtil.type'; -import { OpenapiException } from './openapiCustom.error'; - -const throwOpenapiException = (error: any) => { - if (error.message && error.response && error.response.status) { - throw new OpenapiException( - `Request failed: ${error.message}`, - error.response.status, - error, - ); - } else { - throw new OpenapiException( - `Unknown error: ${error.message || 'No message'}`, - HttpStatus.INTERNAL_SERVER_ERROR, - error, - ); - } -}; - -const postOpenApi = async ( - url: string, - config: typeof openApiConfig, - body: object, -) => { - try { - const response = await axios.post(config.STOCK_URL + url, body); - return response.data; - } catch (error) { - throwOpenapiException(error); - } -}; - -const getOpenApi = async ( - url: string, - config: typeof openApiConfig, - query: object, - tr_id: TR_ID, -) => { - try { - const response = await axios.get(config.STOCK_URL + url, { - params: query, - headers: { - Authorization: `Bearer ${config.STOCK_API_TOKEN}`, - appkey: config.STOCK_API_KEY, - appsecret: config.STOCK_API_PASSWORD, - tr_id, - custtype: 'P', - }, - }); - return response.data; - } catch (error) { - throwOpenapiException(error); - } -}; - -const getTodayDate = (): string => { - const today = new Date(); - return today.toISOString().split('T')[0].replace(/-/g, ''); -}; - -const getPreviousDate = (date: string, months: number): string => { - const currentDate = new Date( - date.slice(0, 4) + '-' + date.slice(4, 6) + '-' + date.slice(6, 8), - ); - currentDate.setMonth(currentDate.getMonth() - months); - return currentDate.toISOString().split('T')[0].replace(/-/g, ''); -}; - -const getCurrentTime = () => { - const now = new Date(); - const hours = String(now.getHours()).padStart(2, '0'); - const minutes = String(now.getMinutes()).padStart(2, '0'); - const seconds = String(now.getSeconds()).padStart(2, '0'); - return `${hours}${minutes}${seconds}`; -}; - -const decryptAES256 = ( - encryptedText: string, - key: string, - iv: string, -): string => { - const decipher = crypto.createDecipheriv( - 'aes-256-cbc', - Buffer.from(key, 'hex'), - Buffer.from(iv, 'hex'), - ); - let decrypted = decipher.update(encryptedText, 'hex', 'utf8'); - decrypted += decipher.final('utf8'); - return decrypted; -}; - -const bufferToObject = (buffer: Buffer): any => { - try { - const jsonString = buffer.toString('utf-8'); - return JSON.parse(jsonString); - } catch (error) { - console.error('Failed to convert buffer to object:', error); - throw error; - } -}; - -export { - postOpenApi, - getOpenApi, - getTodayDate, - getPreviousDate, - getCurrentTime, - decryptAES256, - bufferToObject, -}; diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts b/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts deleted file mode 100644 index a49e5822..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/util/priorityQueue.ts +++ /dev/null @@ -1,99 +0,0 @@ -export class PriorityQueue { - private heap: { value: T; priority: number }[]; - - constructor() { - this.heap = []; - } - - private getParentIndex(index: number): number { - return Math.floor((index - 1) / 2); - } - - private getLeftChildIndex(index: number): number { - return index * 2 + 1; - } - - private getRightChildIndex(index: number): number { - return index * 2 + 2; - } - - private swap(i: number, j: number) { - [this.heap[i], this.heap[j]] = [this.heap[j], this.heap[i]]; - } - - private heapifyUp() { - let index = this.heap.length - 1; - while ( - index > 0 && - this.heap[index].priority < this.heap[this.getParentIndex(index)].priority - ) { - this.swap(index, this.getParentIndex(index)); - index = this.getParentIndex(index); - } - } - - private heapifyDown() { - let index = 0; - while (this.getLeftChildIndex(index) < this.heap.length) { - let smallerChildIndex = this.getLeftChildIndex(index); - const rightChildIndex = this.getRightChildIndex(index); - - if ( - rightChildIndex < this.heap.length && - this.heap[rightChildIndex].priority < - this.heap[smallerChildIndex].priority - ) { - smallerChildIndex = rightChildIndex; - } - - if (this.heap[index].priority <= this.heap[smallerChildIndex].priority) { - break; - } - - this.swap(index, smallerChildIndex); - index = smallerChildIndex; - } - } - - enqueue(value: T, priority: number) { - this.heap.push({ value, priority }); - this.heapifyUp(); - } - - dequeue(): T | undefined { - if (this.isEmpty()) { - return undefined; - } - - const root = this.heap[0]; - const last = this.heap.pop(); - - if (this.heap.length > 0 && last) { - this.heap[0] = last; - this.heapifyDown(); - } - - return root.value; - } - - peek(): T | undefined { - return this.heap.length > 0 ? this.heap[0].value : undefined; - } - - isEmpty(): boolean { - return this.heap.length === 0; - } -} - -const pq = new PriorityQueue(); - -pq.enqueue('Task A', 2); -pq.enqueue('Task B', 1); -pq.enqueue('Task C', 3); - -console.log(pq.dequeue()); // Task B -console.log(pq.peek()); // Task A -console.log(pq.dequeue()); // Task A -console.log(pq.isEmpty()); // false -console.log(pq.dequeue()); // Task C -console.log(pq.isEmpty()); // true diff --git a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts b/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts deleted file mode 100644 index 2e36acfb..00000000 --- a/packages/backend/src/scraper/korea-stock-info/openapi/websocketClient.service.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Inject, Injectable } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { Logger } from 'winston'; -import { RawData, WebSocket } from 'ws'; -import { OpenapiLiveData } from './api/openapiLiveData.api'; -import { openApiToken } from './api/openapiToken.api'; -import { openApiConfig } from './config/openapi.config'; -import { parseMessage } from './parse/openapi.parser'; - -type TR_IDS = '0' | '1'; - -@Injectable() -export class WebsocketClient { - private client: WebSocket; - private readonly reconnectInterval = 60000; - private readonly url = - process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; - private readonly clientStock: Set = new Set(); - - constructor( - @Inject('winston') private readonly logger: Logger, - private readonly openapiLiveData: OpenapiLiveData, - ) { - if (process.env.NODE_ENV === 'production') { - this.connect(); - } - } - - // TODO : subscribe 구조로 리팩토링 - subscribe(stockId: string) { - this.clientStock.add(stockId); - // TODO : 하나의 config만 사용중. - const message = this.convertObjectToMessage( - openApiToken.configs[0], - stockId, - '1', - ); - this.sendMessage(message); - } - - discribe(stockId: string) { - this.clientStock.delete(stockId); - const message = this.convertObjectToMessage( - openApiToken.configs[0], - stockId, - '0', - ); - this.sendMessage(message); - } - - private initDisconnect() { - this.client.on('close', () => { - this.logger.warn( - `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, - ); - }); - - this.client.on('error', (error: any) => { - this.logger.error(`WebSocket error: ${error.message}`); - setTimeout(() => this.connect(), this.reconnectInterval); - }); - } - - private initOpen() { - this.client.on('open', () => { - this.logger.info('WebSocket connection established'); - for (const stockId of this.clientStock.keys()) { - const message = this.convertObjectToMessage( - openApiToken.configs[0], - stockId, - '1', - ); - this.sendMessage(message); - } - }); - } - - private initMessage() { - this.client.on('message', async (data) => { - try { - const message = this.parseMessage(data); - if (message.header) { - if (message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.client.pong({ - tr_id: 'PINGPONG', - datetime: new Date().toISOString(), - }); - } - return; - } - this.logger.info(`Recived data : ${data}`); - const liveData = this.openapiLiveData.convertLiveData(message); - this.openapiLiveData.saveLiveData(liveData); - } catch (error) { - this.logger.warn(error); - } - }); - } - - private parseMessage(data: RawData) { - if (typeof data === 'object') { - return data; - } else { - return parseMessage(data as string); - } - } - - @Cron('0 2 * * 1-5') - connect() { - this.client = new WebSocket(this.url); - this.initOpen(); - this.initMessage(); - this.initDisconnect(); - } - - private convertObjectToMessage( - config: typeof openApiConfig, - stockId: string, - tr_type: TR_IDS, - ): string { - const message = { - header: { - approval_key: config.STOCK_WEBSOCKET_KEY!, - custtype: 'P', - tr_type, - 'content-type': 'utf-8', - }, - body: { - input: { - tr_id: 'H0STCNT0', - tr_key: stockId, - }, - }, - }; - return JSON.stringify(message); - } - - private sendMessage(message: string) { - if (this.client.readyState === WebSocket.OPEN) { - this.client.send(message); - this.logger.info(`Sent message: ${message}`); - } else { - this.logger.warn('WebSocket is not open. Message not sent.'); - } - } -} diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 9729e0fc..16976657 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -1,6 +1,8 @@ +import { Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; +@Injectable() export class OpenapiLiveData { public readonly TR_ID: string = 'H0STCNT0'; constructor(private readonly datasource: DataSource) {} diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index cb45c91c..ad425412 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -39,5 +39,6 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; OpenapiLiveData, WebsocketClient, ], + exports: [WebsocketClient], }) export class OpenapiScraperModule {} diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 2e36acfb..40501a25 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; -import { RawData, WebSocket } from 'ws'; +import { WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; import { openApiToken } from './api/openapiToken.api'; import { openApiConfig } from './config/openapi.config'; @@ -79,7 +79,7 @@ export class WebsocketClient { private initMessage() { this.client.on('message', async (data) => { try { - const message = this.parseMessage(data); + const message = this.parseMessage(data.toString()); if (message.header) { if (message.header.tr_id === 'PINGPONG') { this.logger.info(`Received PING: ${JSON.stringify(data)}`); @@ -99,7 +99,7 @@ export class WebsocketClient { }); } - private parseMessage(data: RawData) { + private parseMessage(data: string) { if (typeof data === 'object') { return data; } else { From 441926fc41b31d480820b331ae84dad070713653 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 22:23:43 +0900 Subject: [PATCH 086/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stock=20gateway?= =?UTF-8?q?=20=EC=88=98=EC=A0=95,=20websocket=20-=20client=20=EC=84=9C?= =?UTF-8?q?=EB=B9=99=20=EA=B5=AC=EC=A1=B0=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/app.module.ts | 8 ++++---- packages/backend/src/stock/stock.gateway.ts | 10 ++++++++-- packages/backend/src/stock/stock.module.ts | 2 ++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/app.module.ts b/packages/backend/src/app.module.ts index f4735ad2..439337b1 100644 --- a/packages/backend/src/app.module.ts +++ b/packages/backend/src/app.module.ts @@ -7,11 +7,11 @@ import { ScraperModule } from './scraper/scraper.module'; import { AuthModule } from '@/auth/auth.module'; import { SessionModule } from '@/auth/session.module'; import { ChatModule } from '@/chat/chat.module'; +import { logger } from '@/configs/logger.config'; import { typeormDevelopConfig, typeormProductConfig, } from '@/configs/typeormConfig'; -import { logger } from '@/configs/logger.config'; import { StockModule } from '@/stock/stock.module'; import { UserModule } from '@/user/user.module'; @@ -19,14 +19,14 @@ import { UserModule } from '@/user/user.module'; imports: [ ConfigModule.forRoot({ cache: true, isGlobal: true }), ScheduleModule.forRoot(), - ScraperModule, - StockModule, - UserModule, TypeOrmModule.forRoot( process.env.NODE_ENV === 'production' ? typeormProductConfig : typeormDevelopConfig, ), + ScraperModule, + StockModule, + UserModule, WinstonModule.forRoot(logger), AuthModule, ChatModule, diff --git a/packages/backend/src/stock/stock.gateway.ts b/packages/backend/src/stock/stock.gateway.ts index 1f4ab32d..c6268e82 100644 --- a/packages/backend/src/stock/stock.gateway.ts +++ b/packages/backend/src/stock/stock.gateway.ts @@ -6,23 +6,29 @@ import { WebSocketServer, } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; +import { WebsocketClient } from '@/scraper/openapi/websocketClient.service'; @WebSocketGateway({ namespace: '/api/stock/realtime', + cors: true, + transports: ['websocket'], }) export class StockGateway { @WebSocketServer() server: Server; - constructor() {} + constructor(private readonly websocketClient: WebsocketClient) {} @SubscribeMessage('connectStock') - handleConnectStock( + async handleConnectStock( @MessageBody() stockId: string, @ConnectedSocket() client: Socket, ) { client.join(stockId); + if ((await this.server.in(stockId).fetchSockets()).length === 0) { + this.websocketClient.subscribe(stockId); + } client.emit('connectionSuccess', { message: `Successfully connected to stock room: ${stockId}`, stockId, diff --git a/packages/backend/src/stock/stock.module.ts b/packages/backend/src/stock/stock.module.ts index 13df81b0..69d925f4 100644 --- a/packages/backend/src/stock/stock.module.ts +++ b/packages/backend/src/stock/stock.module.ts @@ -23,6 +23,7 @@ import { } from './stockData.service'; import { StockDetailService } from './stockDetail.service'; import { StockLiveDataSubscriber } from './stockLiveData.subscriber'; +import { OpenapiScraperModule } from '@/scraper/openapi/openapi-scraper.module'; @Module({ imports: [ @@ -36,6 +37,7 @@ import { StockLiveDataSubscriber } from './stockLiveData.subscriber'; StockLiveData, StockDetail, ]), + OpenapiScraperModule, ], controllers: [StockController], providers: [ From 973733b4d0a4efe1e221632a4706461cf4962a22 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 23:03:02 +0900 Subject: [PATCH 087/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20=EC=A3=BC?= =?UTF-8?q?=EC=8B=9D=20=EC=83=81=EC=84=B8=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?name=20=EC=B6=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/dto/stockDetail.response.ts | 6 ++++++ packages/backend/src/stock/stockDetail.service.ts | 9 ++++++++- packages/backend/src/user/user.service.spec.ts | 10 +++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/stock/dto/stockDetail.response.ts b/packages/backend/src/stock/dto/stockDetail.response.ts index cf692e58..002876c5 100644 --- a/packages/backend/src/stock/dto/stockDetail.response.ts +++ b/packages/backend/src/stock/dto/stockDetail.response.ts @@ -7,6 +7,12 @@ export class StockDetailResponse { }) marketCap: number; + @ApiProperty({ + description: '주식의 이름', + example: '삼성전자', + }) + name: string; + @ApiProperty({ description: '주식의 EPS', example: 4091, diff --git a/packages/backend/src/stock/stockDetail.service.ts b/packages/backend/src/stock/stockDetail.service.ts index 412a4745..65d5a249 100644 --- a/packages/backend/src/stock/stockDetail.service.ts +++ b/packages/backend/src/stock/stockDetail.service.ts @@ -2,6 +2,7 @@ import { Injectable, Inject, NotFoundException } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; import { DataSource } from 'typeorm'; import { Logger } from 'winston'; +import { Stock } from './domain/stock.entity'; import { StockDetail } from './domain/stockDetail.entity'; import { StockDetailResponse } from './dto/stockDetail.response'; @@ -29,7 +30,13 @@ export class StockDetailService { stock: { id: stockId }, }); - return plainToInstance(StockDetailResponse, stockDetail[0]); + const stockName = await manager.findBy(Stock, { + id: stockId, + }); + + const result = { name: stockName[0].name, ...stockDetail[0] }; + + return plainToInstance(StockDetailResponse, result); }); } } diff --git a/packages/backend/src/user/user.service.spec.ts b/packages/backend/src/user/user.service.spec.ts index df80a82c..6af944a1 100644 --- a/packages/backend/src/user/user.service.spec.ts +++ b/packages/backend/src/user/user.service.spec.ts @@ -1,9 +1,9 @@ /* eslint-disable max-lines-per-function */ -import { BadRequestException, NotFoundException } from "@nestjs/common"; -import { DataSource, EntityManager } from "typeorm"; -import { User } from "./domain/user.entity"; -import { OauthType } from "@/user/domain/ouathType"; -import { UserService } from "@/user/user.service"; +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { DataSource, EntityManager } from 'typeorm'; +import { User } from './domain/user.entity'; +import { OauthType } from '@/user/domain/ouathType'; +import { UserService } from '@/user/user.service'; export function createDataSourceMock( managerMock?: Partial, From cdfd1f86c774d929703f53c37f600779b83c5427 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 23:05:37 +0900 Subject: [PATCH 088/112] =?UTF-8?q?Revert=20"=F0=9F=90=9B=20fix:=20stock?= =?UTF-8?q?=20gateway=20=EC=88=98=EC=A0=95,=20websocket=20-=20client=20?= =?UTF-8?q?=EC=84=9C=EB=B9=99=20=EA=B5=AC=EC=A1=B0=20=EB=B3=80=EA=B2=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 441926fc41b31d480820b331ae84dad070713653. --- packages/backend/src/app.module.ts | 8 ++++---- packages/backend/src/stock/stock.gateway.ts | 10 ++-------- packages/backend/src/stock/stock.module.ts | 2 -- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/app.module.ts b/packages/backend/src/app.module.ts index 439337b1..f4735ad2 100644 --- a/packages/backend/src/app.module.ts +++ b/packages/backend/src/app.module.ts @@ -7,11 +7,11 @@ import { ScraperModule } from './scraper/scraper.module'; import { AuthModule } from '@/auth/auth.module'; import { SessionModule } from '@/auth/session.module'; import { ChatModule } from '@/chat/chat.module'; -import { logger } from '@/configs/logger.config'; import { typeormDevelopConfig, typeormProductConfig, } from '@/configs/typeormConfig'; +import { logger } from '@/configs/logger.config'; import { StockModule } from '@/stock/stock.module'; import { UserModule } from '@/user/user.module'; @@ -19,14 +19,14 @@ import { UserModule } from '@/user/user.module'; imports: [ ConfigModule.forRoot({ cache: true, isGlobal: true }), ScheduleModule.forRoot(), + ScraperModule, + StockModule, + UserModule, TypeOrmModule.forRoot( process.env.NODE_ENV === 'production' ? typeormProductConfig : typeormDevelopConfig, ), - ScraperModule, - StockModule, - UserModule, WinstonModule.forRoot(logger), AuthModule, ChatModule, diff --git a/packages/backend/src/stock/stock.gateway.ts b/packages/backend/src/stock/stock.gateway.ts index c6268e82..1f4ab32d 100644 --- a/packages/backend/src/stock/stock.gateway.ts +++ b/packages/backend/src/stock/stock.gateway.ts @@ -6,29 +6,23 @@ import { WebSocketServer, } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; -import { WebsocketClient } from '@/scraper/openapi/websocketClient.service'; @WebSocketGateway({ namespace: '/api/stock/realtime', - cors: true, - transports: ['websocket'], }) export class StockGateway { @WebSocketServer() server: Server; - constructor(private readonly websocketClient: WebsocketClient) {} + constructor() {} @SubscribeMessage('connectStock') - async handleConnectStock( + handleConnectStock( @MessageBody() stockId: string, @ConnectedSocket() client: Socket, ) { client.join(stockId); - if ((await this.server.in(stockId).fetchSockets()).length === 0) { - this.websocketClient.subscribe(stockId); - } client.emit('connectionSuccess', { message: `Successfully connected to stock room: ${stockId}`, stockId, diff --git a/packages/backend/src/stock/stock.module.ts b/packages/backend/src/stock/stock.module.ts index 69d925f4..13df81b0 100644 --- a/packages/backend/src/stock/stock.module.ts +++ b/packages/backend/src/stock/stock.module.ts @@ -23,7 +23,6 @@ import { } from './stockData.service'; import { StockDetailService } from './stockDetail.service'; import { StockLiveDataSubscriber } from './stockLiveData.subscriber'; -import { OpenapiScraperModule } from '@/scraper/openapi/openapi-scraper.module'; @Module({ imports: [ @@ -37,7 +36,6 @@ import { OpenapiScraperModule } from '@/scraper/openapi/openapi-scraper.module'; StockLiveData, StockDetail, ]), - OpenapiScraperModule, ], controllers: [StockController], providers: [ From 9b395d3792b6e6f8bddcb989233c133b5c6e73ec Mon Sep 17 00:00:00 2001 From: sunghwki Date: Sun, 24 Nov 2024 23:31:44 +0900 Subject: [PATCH 089/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20stock?= =?UTF-8?q?=20detail=20=EA=B8=B0=EB=B3=B8=EC=BF=BC=EB=A6=AC=EA=B0=80=20?= =?UTF-8?q?=EC=95=84=EB=8B=8C=20left=20join=20=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/stock/dto/stockDetail.response.ts | 10 ++++++++ .../backend/src/stock/stockDetail.service.ts | 23 ++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/stock/dto/stockDetail.response.ts b/packages/backend/src/stock/dto/stockDetail.response.ts index 002876c5..248163fd 100644 --- a/packages/backend/src/stock/dto/stockDetail.response.ts +++ b/packages/backend/src/stock/dto/stockDetail.response.ts @@ -1,4 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; +import { StockDetail } from '../domain/stockDetail.entity'; export class StockDetailResponse { @ApiProperty({ @@ -36,4 +37,13 @@ export class StockDetailResponse { example: 53000, }) low52w: number; + + constructor(stockDetail: StockDetail) { + this.eps = stockDetail.eps; + this.per = stockDetail.per; + this.high52w = stockDetail.high52w; + this.low52w = stockDetail.low52w; + this.marketCap = Number(stockDetail.marketCap); + this.name = stockDetail.stock.name; + } } diff --git a/packages/backend/src/stock/stockDetail.service.ts b/packages/backend/src/stock/stockDetail.service.ts index 65d5a249..828a40ee 100644 --- a/packages/backend/src/stock/stockDetail.service.ts +++ b/packages/backend/src/stock/stockDetail.service.ts @@ -1,8 +1,6 @@ import { Injectable, Inject, NotFoundException } from '@nestjs/common'; -import { plainToInstance } from 'class-transformer'; import { DataSource } from 'typeorm'; import { Logger } from 'winston'; -import { Stock } from './domain/stock.entity'; import { StockDetail } from './domain/stockDetail.entity'; import { StockDetailResponse } from './dto/stockDetail.response'; @@ -26,17 +24,20 @@ export class StockDetailService { ); } - const stockDetail = await manager.findBy(StockDetail, { - stock: { id: stockId }, - }); - - const stockName = await manager.findBy(Stock, { - id: stockId, - }); + const result = await this.datasource.manager + .getRepository(StockDetail) + .createQueryBuilder('stockDetail') + .leftJoinAndSelect('stockDetail.stock', 'stock') + .where('stockDetail.stock_id = :stockId', { stockId }) + .getOne(); - const result = { name: stockName[0].name, ...stockDetail[0] }; + if (!result) { + throw new NotFoundException( + `stock detail not found (stockId: ${stockId}`, + ); + } - return plainToInstance(StockDetailResponse, result); + return new StockDetailResponse(result); }); } } From b71839a219595cee9e6116d28d9dd36d69ab0517 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 25 Nov 2024 11:37:06 +0900 Subject: [PATCH 090/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=9C=A0=EC=A0=80?= =?UTF-8?q?=20=EB=9E=9C=EB=8D=A4=20=EB=8B=89=EB=84=A4=EC=9E=84=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/user/user.service.ts | 32 ++++------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/user/user.service.ts b/packages/backend/src/user/user.service.ts index 8d2a5d23..dbef92dd 100644 --- a/packages/backend/src/user/user.service.ts +++ b/packages/backend/src/user/user.service.ts @@ -6,6 +6,7 @@ import { import { DataSource, EntityManager } from 'typeorm'; import { OauthType } from './domain/ouathType'; import { User } from './domain/user.entity'; +import { status, subject } from '@/user/constants/randomNickname'; type RegisterRequest = Required< Pick @@ -76,33 +77,10 @@ export class UserService { return user.isLight; } - private generateRandomNickname(): string { - const adjectives = [ - '강력한', - '지혜로운', - '소중한', - '빛나는', - '고요한', - '용감한', - '행운의', - '신비로운', - ]; - const animals = [ - '호랑이', - '독수리', - '용', - '사슴', - '백호', - '하늘새', - '백두산 호랑이', - '붉은 여우', - ]; - - const randomAdjective = - adjectives[Math.floor(Math.random() * adjectives.length)]; - const randomAnimal = animals[Math.floor(Math.random() * animals.length)]; - - return `${randomAdjective} ${randomAnimal}`; + private generateRandomNickname() { + const statusName = status[Math.floor(Math.random() * status.length)]; + const subjectName = subject[Math.floor(Math.random() * subject.length)]; + return `${statusName}${subjectName}`; } private async getMaxOauthId(oauthType: OauthType, manager: EntityManager) { From 22d60c65f57beda606df335a92c84e8bb220b9b0 Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 25 Nov 2024 11:40:48 +0900 Subject: [PATCH 091/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=EB=A1=9C=EA=B7=B8=EC=9D=B8=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20swagger=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/auth/tester/testerAuth.controller.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/auth/tester/testerAuth.controller.ts b/packages/backend/src/auth/tester/testerAuth.controller.ts index d4381db9..4ee02553 100644 --- a/packages/backend/src/auth/tester/testerAuth.controller.ts +++ b/packages/backend/src/auth/tester/testerAuth.controller.ts @@ -1,5 +1,10 @@ import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; -import { ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { + ApiOkResponse, + ApiOperation, + ApiQuery, + ApiTags, +} from '@nestjs/swagger'; import { Request, Response } from 'express'; import { TestAuthGuard } from '@/auth/tester/guard/tester.guard'; @@ -12,6 +17,16 @@ export class TesterAuthController { summary: '테스터 로그인 api', description: '테스터로 로그인합니다.', }) + @ApiQuery({ + name: 'username', + required: true, + description: '테스터 아이디(값만 넣으면 됨)', + }) + @ApiQuery({ + name: 'password', + required: true, + description: '테스터 비밀번호(값만 넣으면 됨)', + }) @Get('/login') @UseGuards(TestAuthGuard) async handleLogin(@Res() response: Response) { From 8fe29efeae38aab7f01fb7527098c112907336c9 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 13:39:08 +0900 Subject: [PATCH 092/112] =?UTF-8?q?=E2=9C=A8=20feat:=20token=20entity=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/domain/openapiToken.entity.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 packages/backend/src/scraper/domain/openapiToken.entity.ts diff --git a/packages/backend/src/scraper/domain/openapiToken.entity.ts b/packages/backend/src/scraper/domain/openapiToken.entity.ts new file mode 100644 index 00000000..613abcb5 --- /dev/null +++ b/packages/backend/src/scraper/domain/openapiToken.entity.ts @@ -0,0 +1,28 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'openapi_token' }) +export class OpenapiToken { + @PrimaryColumn({ name: 'account' }) + account: string; + + @Column({ name: 'apiUrl' }) + api_url: string; + + @Column({ name: 'key' }) + api_key: string; + + @Column({ name: 'password' }) + api_password: string; + + @Column({ name: 'token', length: 512 }) + api_token?: string; + + @Column({ name: 'tokenExpire' }) + api_token_expire?: Date; + + @Column({ name: 'websocketKey' }) + websocket_key?: string; + + @Column({ name: 'websocketKeyExpire' }) + websocket_key_expire?: Date; +} From 903264e45da03548a7c91e75f455e65947c91be4 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 13:39:47 +0900 Subject: [PATCH 093/112] =?UTF-8?q?=E2=9C=A8=20feat:=20entity=20=EC=97=90?= =?UTF-8?q?=20=EC=A0=80=EC=9E=A5,=20expire=20=EA=B2=80=EC=82=AC=20?= =?UTF-8?q?=EB=A1=9C=EC=A7=81=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scraper/openapi/api/openapiToken.api.ts | 128 ++++++++++++++++-- 1 file changed, 115 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 6e6c88c5..545ee4e1 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,14 +1,21 @@ -import { Inject } from '@nestjs/common'; +import { Global, Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; +import { DataSource } from 'typeorm'; import { Logger } from 'winston'; +import { OpenapiToken } from '../../domain/openapiToken.entity'; import { openApiConfig } from '../config/openapi.config'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; -import { logger } from '@/configs/logger.config'; -class OpenapiTokenApi { +@Global() +@Injectable() +export class OpenapiTokenApi { private config: (typeof openApiConfig)[] = []; - constructor(@Inject('winston') private readonly logger: Logger) { + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly datasource: DataSource, + ) { + if (process.env.NODE_ENV !== 'production') return; const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); const api_keys = openApiConfig.STOCK_API_KEY!.split(','); const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); @@ -27,14 +34,114 @@ class OpenapiTokenApi { STOCK_API_PASSWORD: api_passwords[i], }); } - this.initAuthenValue(); + this.init(); } get configs() { - //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 + this.init(); return this.config; } + @Cron('30 0 * * 1-5') + async init() { + const tokens = await this.convertConfigToTokenEntity(this.config); + const config = await this.getPropertyFromDB(tokens); + const expired = config.filter( + (val) => + this.isTokenExpired(val.api_token_expire) && + this.isTokenExpired(val.websocket_key_expire), + ); + + if (expired.length || !config.length) { + console.log('in if'); + console.log(expired); + console.log(config); + await this.initAuthenValue(); + const newTokens = await this.convertConfigToTokenEntity(this.config); + this.savePropertyToDB(newTokens); + } else { + this.config = await this.convertTokenEntityToConfig(config); + } + } + + private isTokenExpired(startDate?: Date) { + if (!startDate) return true; + const now = new Date(); + //실제 만료 시간은 24시간이지만, 문제의 소지가 발생하는 것을 방지하기 위해 20시간으로 설정함. + const baseTimeToMilliSec = 20 * 60 * 60 * 1000; + const timeDiff = now.getTime() - startDate.getTime(); + + return timeDiff >= baseTimeToMilliSec; + } + + private async convertTokenEntityToConfig(tokens: OpenapiToken[]) { + const result: (typeof openApiConfig)[] = []; + tokens.forEach((val) => { + const config: typeof openApiConfig = { + STOCK_ACCOUNT: val.account, + STOCK_API_KEY: val.api_key, + STOCK_API_PASSWORD: val.api_password, + STOCK_API_TOKEN: val.api_token, + STOCK_URL: val.api_url, + STOCK_WEBSOCKET_KEY: val.websocket_key, + }; + result.push(config); + }); + return result; + } + + private async convertConfigToTokenEntity(config: (typeof openApiConfig)[]) { + const result: OpenapiToken[] = []; + config.forEach((val) => { + const token = new OpenapiToken(); + if ( + val.STOCK_URL && + val.STOCK_ACCOUNT && + val.STOCK_API_KEY && + val.STOCK_API_PASSWORD + ) { + token.api_url = val.STOCK_URL; + token.account = val.STOCK_ACCOUNT; + token.api_key = val.STOCK_API_KEY; + token.api_password = val.STOCK_API_PASSWORD; + } + token.api_token = val.STOCK_API_TOKEN; + token.websocket_key = val.STOCK_WEBSOCKET_KEY; + token.api_token_expire = new Date(); + token.websocket_key_expire = new Date(); + result.push(token); + }); + return result; + } + + private async savePropertyToDB(tokens: OpenapiToken[]) { + tokens.forEach(async (val) => { + this.datasource.manager.save(OpenapiToken, val); + }); + } + + private async getPropertyFromDB(tokens: OpenapiToken[]) { + const result: OpenapiToken[] = []; + await Promise.all( + tokens.map(async (val) => { + const findByToken = await this.datasource.manager.findOne( + OpenapiToken, + { + where: { + account: val.account, + api_key: val.api_key, + api_password: val.api_password, + }, + }, + ); + if (findByToken) { + result.push(findByToken); + } + }), + ); + return result; + } + private async initAuthenValue() { const delay = 60000; const delayMinute = delay / 1000 / 60; @@ -59,8 +166,7 @@ class OpenapiTokenApi { } } - @Cron('50 0 * * 1-5') - async initAccessToken() { + private async initAccessToken() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_API_TOKEN = await this.getToken(val)!; @@ -70,8 +176,7 @@ class OpenapiTokenApi { this.config = updatedConfig; } - @Cron('50 0 * * 1-5') - async initWebSocketKey() { + private async initWebSocketKey() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; @@ -107,6 +212,3 @@ class OpenapiTokenApi { return tmp.approval_key as string; } } - -const openApiToken = new OpenapiTokenApi(logger); -export { openApiToken }; From e4429e5bf390e0b11e4d7e8952709b7d2ddfc09f Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 13:40:25 +0900 Subject: [PATCH 094/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20token=20=EC=A3=BC?= =?UTF-8?q?=EC=9E=85=EC=9C=BC=EB=A1=9C=20=EB=A1=9C=EC=A7=81=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/openapi/api/openapiDetailData.api.ts | 9 +++++---- .../src/scraper/openapi/api/openapiMinuteData.api.ts | 11 ++++++----- .../src/scraper/openapi/api/openapiPeriodData.api.ts | 9 +++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 9be8e3ea..6a32435f 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -13,7 +13,7 @@ import { } from '../type/openapiDetailData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { KospiStock } from '@/stock/domain/kospiStock.entity'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; @@ -27,6 +27,7 @@ export class OpenapiDetailData { '/uapi/domestic-stock/v1/quotations/search-stock-info'; private readonly intervals = 1000; constructor( + private readonly openApiToken: OpenapiTokenApi, private readonly datasource: DataSource, @Inject('winston') private readonly logger: Logger, ) { @@ -38,13 +39,13 @@ export class OpenapiDetailData { if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { - this.logger.info(openApiToken.configs[i]); + this.logger.info(this.openApiToken.configs[i]); const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, openApiToken.configs[i]); + this.getDetailDataChunk(chunk, this.openApiToken.configs[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index 003e7560..e64e26c7 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -11,7 +11,7 @@ import { } from '../type/openapiMinuteData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; @@ -27,9 +27,10 @@ export class OpenapiMinuteData { private flip: number = 0; constructor( private readonly datasource: DataSource, + private readonly openApiToken: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) { - this.getStockData(); + //this.getStockData(); } @Cron('0 1 * * 1-5') @@ -126,16 +127,16 @@ export class OpenapiMinuteData { } } - @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) + //@Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) getMinuteData() { if (process.env.NODE_ENV !== 'production') return; - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const stock = this.stock[this.flip % STOCK_CUT]; this.flip++; const chunkSize = Math.ceil(stock.length / configCount); for (let i = 0; i < configCount; i++) { const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); - this.getMinuteDataChunk(chunk, openApiToken.configs[i]); + this.getMinuteDataChunk(chunk, this.openApiToken.configs[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index f4268088..4b976be8 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -14,7 +14,7 @@ import { getPreviousDate, getTodayDate, } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, @@ -46,6 +46,7 @@ export class OpenapiPeriodData { '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; constructor( private readonly datasource: DataSource, + private readonly openApiToken: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) { //this.getItemChartPriceCheck(); @@ -59,7 +60,7 @@ export class OpenapiPeriodData { isTrading: true, }, }); - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { @@ -95,7 +96,7 @@ export class OpenapiPeriodData { let isFail = false; while (!isFail) { - configIdx = (configIdx + 1) % openApiToken.configs.length; + configIdx = (configIdx + 1) % this.openApiToken.configs.length; this.setStockPeriod(stockPeriod, stock.id!, end); // chart 데이터가 있는 지 확인 -> 리턴 @@ -129,7 +130,7 @@ export class OpenapiPeriodData { try { const response = await getOpenApi( this.url, - openApiToken.configs[configIdx], + this.openApiToken.configs[configIdx], query, TR_IDS.ITEM_CHART_PRICE, ); From 3a81ad036ac4d5aa900d91e583d9a79d72a15265 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 13:41:01 +0900 Subject: [PATCH 095/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20token?= =?UTF-8?q?=20=EC=A3=BC=EC=9E=85=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD,?= =?UTF-8?q?=20=EA=B7=B8=EB=A1=9C=20=EC=9D=B8=ED=95=9C=20=EC=98=A4=EB=A5=98?= =?UTF-8?q?=20=EC=88=98=EC=A0=95=20=EB=B0=8F=20console.log=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scraper/openapi/openapi-scraper.module.ts | 3 ++ .../src/scraper/openapi/util/priorityQueue.ts | 13 -------- .../openapi/websocketClient.service.ts | 30 ++++++++++--------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index ad425412..1107dbb9 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,9 +1,11 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { OpenapiToken } from '../domain/openapiToken.entity'; import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; +import { OpenapiTokenApi } from './api/openapiToken.api'; import { OpenapiScraperService } from './openapi-scraper.service'; import { WebsocketClient } from './websocketClient.service'; import { Stock } from '@/stock/domain/stock.entity'; @@ -32,6 +34,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; ], controllers: [], providers: [ + OpenapiTokenApi, OpenapiPeriodData, OpenapiMinuteData, OpenapiDetailData, diff --git a/packages/backend/src/scraper/openapi/util/priorityQueue.ts b/packages/backend/src/scraper/openapi/util/priorityQueue.ts index a49e5822..190718a5 100644 --- a/packages/backend/src/scraper/openapi/util/priorityQueue.ts +++ b/packages/backend/src/scraper/openapi/util/priorityQueue.ts @@ -84,16 +84,3 @@ export class PriorityQueue { return this.heap.length === 0; } } - -const pq = new PriorityQueue(); - -pq.enqueue('Task A', 2); -pq.enqueue('Task B', 1); -pq.enqueue('Task C', 3); - -console.log(pq.dequeue()); // Task B -console.log(pq.peek()); // Task A -console.log(pq.dequeue()); // Task A -console.log(pq.isEmpty()); // false -console.log(pq.dequeue()); // Task C -console.log(pq.isEmpty()); // true diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 40501a25..1533f251 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -2,9 +2,9 @@ import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; -import { WebSocket } from 'ws'; +import { RawData, WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; -import { openApiToken } from './api/openapiToken.api'; +import { OpenapiTokenApi } from './api/openapiToken.api'; import { openApiConfig } from './config/openapi.config'; import { parseMessage } from './parse/openapi.parser'; @@ -20,10 +20,12 @@ export class WebsocketClient { constructor( @Inject('winston') private readonly logger: Logger, + private readonly openApiToken: OpenapiTokenApi, private readonly openapiLiveData: OpenapiLiveData, ) { if (process.env.NODE_ENV === 'production') { this.connect(); + setTimeout(() => this.subscribe('000150'), 5000); } } @@ -32,7 +34,7 @@ export class WebsocketClient { this.clientStock.add(stockId); // TODO : 하나의 config만 사용중. const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '1', ); @@ -42,7 +44,7 @@ export class WebsocketClient { discribe(stockId: string) { this.clientStock.delete(stockId); const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '0', ); @@ -67,7 +69,7 @@ export class WebsocketClient { this.logger.info('WebSocket connection established'); for (const stockId of this.clientStock.keys()) { const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '1', ); @@ -77,16 +79,14 @@ export class WebsocketClient { } private initMessage() { - this.client.on('message', async (data) => { + this.client.on('message', async (data: RawData) => { try { - const message = this.parseMessage(data.toString()); + console.log(data); + const message = this.parseMessage(data); if (message.header) { if (message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.client.pong({ - tr_id: 'PINGPONG', - datetime: new Date().toISOString(), - }); + this.logger.info(`Received PING: ${data}`); + this.client.pong(data); } return; } @@ -99,9 +99,11 @@ export class WebsocketClient { }); } - private parseMessage(data: string) { - if (typeof data === 'object') { + private parseMessage(data: RawData) { + if (typeof data === 'object' && !(data instanceof Buffer)) { return data; + } else if (typeof data === 'object') { + return parseMessage(data.toString()); } else { return parseMessage(data as string); } From 57f1d5ad4490b561dd62f598ac14d0f0dcc6d5d7 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 14:47:03 +0900 Subject: [PATCH 096/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20live=20=EB=8D=B0?= =?UTF-8?q?=EC=9D=B4=ED=84=B0=20=EC=88=98=EC=A7=91=20=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0,=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=97=86?= =?UTF-8?q?=EC=9D=84=20=EB=95=8C=20insert=20=EC=98=A4=EB=A5=98=20=ED=95=B4?= =?UTF-8?q?=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiLiveData.api.ts | 39 ++++++++++++++----- .../scraper/openapi/api/openapiToken.api.ts | 4 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 16976657..6fa15505 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -1,26 +1,47 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { Stock } from '@/stock/domain/stock.entity'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @Injectable() export class OpenapiLiveData { public readonly TR_ID: string = 'H0STCNT0'; - constructor(private readonly datasource: DataSource) {} + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) {} async saveLiveData(data: StockLiveData[]) { - await this.datasource.manager - .getRepository(StockLiveData) - .createQueryBuilder() - .insert() - .into(StockLiveData) - .values(data) - .execute(); + const exists = await this.datasource.manager.exists(StockLiveData, { + where: { + stock: { id: data[0].stock.id }, + }, + }); + if (exists) { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .update() + .set(data[0]) + .where('stock.id = :stockId', { stockId: data[0].stock.id }) + .execute(); + } else { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .insert() + .into(StockLiveData) + .values(data) + .execute(); + } } convertLiveData(messages: Record[]): StockLiveData[] { const stockData: StockLiveData[] = []; messages.map((message) => { const stockLiveData = new StockLiveData(); + stockLiveData.stock = { id: message.STOCK_ID } as Stock; stockLiveData.currentPrice = parseFloat(message.STCK_PRPR); stockLiveData.changeRate = parseFloat(message.PRDY_CTRT); stockLiveData.volume = parseInt(message.CNTG_VOL); diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 545ee4e1..8d812824 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -53,9 +53,7 @@ export class OpenapiTokenApi { ); if (expired.length || !config.length) { - console.log('in if'); - console.log(expired); - console.log(config); + await this.initAuthenValue(); const newTokens = await this.convertConfigToTokenEntity(this.config); this.savePropertyToDB(newTokens); From 1de39b499fd62c1065f031a0bc73cc4b6bd16bdd Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 14:47:46 +0900 Subject: [PATCH 097/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stock,=20livedata?= =?UTF-8?q?=20entity=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/domain/stock.entity.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index be321f55..994a6de1 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -7,6 +7,7 @@ import { StockWeekly, StockYearly, } from './stockData.entity'; +import { StockLiveData } from './stockLiveData.entity'; import { Like } from '@/chat/domain/like.entity'; import { DateEmbedded } from '@/common/dateEmbedded.entity'; import { UserStock } from '@/stock/domain/userStock.entity'; @@ -52,6 +53,9 @@ export class Stock { @OneToMany(() => StockYearly, (stockYearly) => stockYearly.stock) stockYearly?: StockYearly[]; + @OneToOne(() => StockLiveData, (stockLiveData) => stockLiveData.stock) + stockLive?: StockLiveData; + @OneToOne(() => KospiStock, (kospiStock) => kospiStock.stock) kospiStock?: KospiStock; } From bb711259f675e77670be7a3e2689fe5bf93dc796 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 14:48:15 +0900 Subject: [PATCH 098/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20develo?= =?UTF-8?q?p=20=ED=99=98=EA=B2=BD=EC=8B=9C=20logging=20=ED=99=9C=EC=84=B1?= =?UTF-8?q?=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/typeormConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/configs/typeormConfig.ts b/packages/backend/src/configs/typeormConfig.ts index a8c70a18..f8ff59ac 100644 --- a/packages/backend/src/configs/typeormConfig.ts +++ b/packages/backend/src/configs/typeormConfig.ts @@ -21,6 +21,6 @@ export const typeormDevelopConfig: TypeOrmModuleOptions = { password: process.env.DB_PASS, database: process.env.DB_NAME, entities: [__dirname + '/../**/*.entity.{js,ts}'], - //logging: true, + logging: true, synchronize: true, }; From e7e67247eeb40fdff35e1261be5c2d9eaa4333fb Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 14:48:55 +0900 Subject: [PATCH 099/112] =?UTF-8?q?=F0=9F=93=A6=EF=B8=8F=20ci:=20productio?= =?UTF-8?q?n=20=ED=99=98=EA=B2=BD=EC=9D=BC=20=EB=95=8C=20=EC=9E=91?= =?UTF-8?q?=EB=8F=99=EB=90=98=EA=B2=8C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/api/openapiToken.api.ts | 1 - .../backend/src/scraper/openapi/openapi-scraper.module.ts | 1 - .../backend/src/scraper/openapi/websocketClient.service.ts | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 8d812824..829ca9a3 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -53,7 +53,6 @@ export class OpenapiTokenApi { ); if (expired.length || !config.length) { - await this.initAuthenValue(); const newTokens = await this.convertConfigToTokenEntity(this.config); this.savePropertyToDB(newTokens); diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 1107dbb9..73f33f23 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,6 +1,5 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { OpenapiToken } from '../domain/openapiToken.entity'; import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 1533f251..e784afe5 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -25,7 +25,6 @@ export class WebsocketClient { ) { if (process.env.NODE_ENV === 'production') { this.connect(); - setTimeout(() => this.subscribe('000150'), 5000); } } @@ -91,8 +90,9 @@ export class WebsocketClient { return; } this.logger.info(`Recived data : ${data}`); + this.logger.info(`Stock id : ${message[0]['STOCK_ID']}`); const liveData = this.openapiLiveData.convertLiveData(message); - this.openapiLiveData.saveLiveData(liveData); + await this.openapiLiveData.saveLiveData(liveData); } catch (error) { this.logger.warn(error); } From 996b64a52890fc4ee1e7cd6ee1ee3be582e4407c Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 25 Nov 2024 17:02:23 +0900 Subject: [PATCH 100/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EB=9E=9C=EB=8D=A4?= =?UTF-8?q?=20=EB=8B=89=EB=84=A4=EC=9E=84=20=EC=83=81=EC=88=98=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/user/constants/randomNickname.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 packages/backend/src/user/constants/randomNickname.ts diff --git a/packages/backend/src/user/constants/randomNickname.ts b/packages/backend/src/user/constants/randomNickname.ts new file mode 100644 index 00000000..fb55e86e --- /dev/null +++ b/packages/backend/src/user/constants/randomNickname.ts @@ -0,0 +1,16 @@ +export const status = [ + '신중한', + '과감한', + '공부하는', + '성장하는', + '주춤거리는', +]; +export const subject = [ + '병아리', + '햄스터', + '다람쥐', + '거북이', + '판다', + '주린이', + '투자자', +]; From 8f43995d57fe2acb96e1c18712dbb157c32e634d Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 17:50:24 +0900 Subject: [PATCH 101/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20websoc?= =?UTF-8?q?ket=20=EB=AA=A8=EB=93=88=EC=97=90=EC=84=9C=20liveData=EB=A1=9C?= =?UTF-8?q?=20=EC=84=9C=EB=B9=84=EC=8A=A4=20=EB=A1=9C=EC=A7=81=20=EB=B6=84?= =?UTF-8?q?=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/api/openapiDetailData.api.ts | 6 +- .../openapi/api/openapiLiveData.api.ts | 30 ++++- .../openapi/api/openapiMinuteData.api.ts | 6 +- .../openapi/api/openapiPeriodData.api.ts | 4 +- .../scraper/openapi/api/openapiToken.api.ts | 5 +- ...tClient.service.ts => liveData.service.ts} | 106 +++++++++--------- .../scraper/openapi/openapi-scraper.module.ts | 4 +- .../websocket/websocketClient.websocket.ts | 59 ++++++++++ packages/backend/src/stock/stock.gateway.ts | 2 +- 9 files changed, 152 insertions(+), 70 deletions(-) rename packages/backend/src/scraper/openapi/{websocketClient.service.ts => liveData.service.ts} (63%) create mode 100644 packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 6a32435f..982ebaf6 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -39,13 +39,13 @@ export class OpenapiDetailData { if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); - const configCount = this.openApiToken.configs.length; + const configCount = (await this.openApiToken.configs()).length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { - this.logger.info(this.openApiToken.configs[i]); + this.logger.info((await this.openApiToken.configs())[i]); const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, this.openApiToken.configs[i]); + this.getDetailDataChunk(chunk, (await this.openApiToken.configs())[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 6fa15505..27375f83 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -1,14 +1,19 @@ import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Logger } from 'winston'; +import { TR_IDS } from '../type/openapiUtil.type'; +import { getOpenApi } from '../util/openapiUtil.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @Injectable() export class OpenapiLiveData { - public readonly TR_ID: string = 'H0STCNT0'; + private readonly url: string = + '/uapi/domestic-stock/v1/quotations/inquire-ccnl'; constructor( private readonly datasource: DataSource, + private readonly config: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) {} @@ -54,4 +59,27 @@ export class OpenapiLiveData { }); return stockData; } + + async connectLiveData(stockId: string) { + const query = this.makeLiveDataQuery(stockId); + + try { + const result = await getOpenApi( + this.url, + (await this.config.configs())[0], + query, + TR_IDS.ITEM_CHART_PRICE, + ); + return result; + } catch (error) { + this.logger.warn(`Connect live data error : ${error}`); + } + } + + private makeLiveDataQuery(stockId: string, code: 'J' = 'J') { + return { + fid_cond_mrkt_div_code: code, + fid_input_iscd: stockId, + }; + } } diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index e64e26c7..4e193b74 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -128,15 +128,15 @@ export class OpenapiMinuteData { } //@Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) - getMinuteData() { + async getMinuteData() { if (process.env.NODE_ENV !== 'production') return; - const configCount = this.openApiToken.configs.length; + const configCount = (await this.openApiToken.configs()).length; const stock = this.stock[this.flip % STOCK_CUT]; this.flip++; const chunkSize = Math.ceil(stock.length / configCount); for (let i = 0; i < configCount; i++) { const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); - this.getMinuteDataChunk(chunk, this.openApiToken.configs[i]); + this.getMinuteDataChunk(chunk, (await this.openApiToken.configs())[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 4b976be8..5471cf5e 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -96,7 +96,7 @@ export class OpenapiPeriodData { let isFail = false; while (!isFail) { - configIdx = (configIdx + 1) % this.openApiToken.configs.length; + configIdx = (configIdx + 1) % (await this.openApiToken.configs()).length; this.setStockPeriod(stockPeriod, stock.id!, end); // chart 데이터가 있는 지 확인 -> 리턴 @@ -130,7 +130,7 @@ export class OpenapiPeriodData { try { const response = await getOpenApi( this.url, - this.openApiToken.configs[configIdx], + (await this.openApiToken.configs())[configIdx], query, TR_IDS.ITEM_CHART_PRICE, ); diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 829ca9a3..deedce9b 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -15,7 +15,6 @@ export class OpenapiTokenApi { @Inject('winston') private readonly logger: Logger, private readonly datasource: DataSource, ) { - if (process.env.NODE_ENV !== 'production') return; const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); const api_keys = openApiConfig.STOCK_API_KEY!.split(','); const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); @@ -37,8 +36,8 @@ export class OpenapiTokenApi { this.init(); } - get configs() { - this.init(); + async configs() { + await this.init(); return this.config; } diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts similarity index 63% rename from packages/backend/src/scraper/openapi/websocketClient.service.ts rename to packages/backend/src/scraper/openapi/liveData.service.ts index e784afe5..a4825a18 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -1,4 +1,5 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ +//TODO : 9시 ~ 3시 반까지는 openapi에서 가져오고, 아니면 websocket으로 가져오기 + import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; @@ -7,85 +8,70 @@ import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiTokenApi } from './api/openapiToken.api'; import { openApiConfig } from './config/openapi.config'; import { parseMessage } from './parse/openapi.parser'; +import { WebsocketClient } from './websocket/websocketClient.websocket'; type TR_IDS = '0' | '1'; - +// TODO : 비즈니스 로직을 분리해야함. @Injectable() -export class WebsocketClient { - private client: WebSocket; - private readonly reconnectInterval = 60000; +export class LiveData { private readonly url = process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; private readonly clientStock: Set = new Set(); + private readonly reconnectInterval = 60 * 1000 * 1000; constructor( - @Inject('winston') private readonly logger: Logger, private readonly openApiToken: OpenapiTokenApi, + private readonly webSocketClient: WebsocketClient, private readonly openapiLiveData: OpenapiLiveData, + @Inject('winston') private readonly logger: Logger, ) { - if (process.env.NODE_ENV === 'production') { - this.connect(); - } + this.connect(); + this.subscribe('000150'); } - // TODO : subscribe 구조로 리팩토링 - subscribe(stockId: string) { + async subscribe(stockId: string) { this.clientStock.add(stockId); // TODO : 하나의 config만 사용중. const message = this.convertObjectToMessage( - this.openApiToken.configs[0], + (await this.openApiToken.configs())[0], stockId, '1', ); - this.sendMessage(message); + this.webSocketClient.subscribe(message); } - discribe(stockId: string) { + async discribe(stockId: string) { this.clientStock.delete(stockId); const message = this.convertObjectToMessage( - this.openApiToken.configs[0], + (await this.openApiToken.configs())[0], stockId, '0', ); - this.sendMessage(message); + this.webSocketClient.discribe(message); } - private initDisconnect() { - this.client.on('close', () => { - this.logger.warn( - `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, - ); - }); - - this.client.on('error', (error: any) => { - this.logger.error(`WebSocket error: ${error.message}`); - setTimeout(() => this.connect(), this.reconnectInterval); - }); - } - - private initOpen() { - this.client.on('open', () => { + private initOpenCallback = + (sendMessage: (message: string) => void) => async () => { this.logger.info('WebSocket connection established'); for (const stockId of this.clientStock.keys()) { const message = this.convertObjectToMessage( - this.openApiToken.configs[0], + (await this.openApiToken.configs())[0], stockId, '1', ); - this.sendMessage(message); + sendMessage(message); } - }); - } + }; - private initMessage() { - this.client.on('message', async (data: RawData) => { + private initMessageCallback = + (client: WebSocket) => async (data: RawData) => { try { console.log(data); const message = this.parseMessage(data); if (message.header) { if (message.header.tr_id === 'PINGPONG') { this.logger.info(`Received PING: ${data}`); - this.client.pong(data); + client.pong(data); } return; } @@ -96,25 +82,31 @@ export class WebsocketClient { } catch (error) { this.logger.warn(error); } - }); - } + }; - private parseMessage(data: RawData) { - if (typeof data === 'object' && !(data instanceof Buffer)) { - return data; - } else if (typeof data === 'object') { - return parseMessage(data.toString()); + private initCloseCallback = () => { + this.logger.warn( + `WebSocket connection closed. Reconnecting in ${this.reconnectInterval / 60 / 1000} minute...`, + ); + }; + + private initErrorCallback = (error: unknown) => { + if (error instanceof Error) { + this.logger.error(`WebSocket error: ${error.message}`); } else { - return parseMessage(data as string); + this.logger.error('WebSocket error: callback function'); } - } + setTimeout(() => this.connect(), this.reconnectInterval); + }; @Cron('0 2 * * 1-5') connect() { - this.client = new WebSocket(this.url); - this.initOpen(); - this.initMessage(); - this.initDisconnect(); + this.webSocketClient.connect( + this.initOpenCallback, + this.initMessageCallback, + this.initCloseCallback, + this.initErrorCallback, + ); } private convertObjectToMessage( @@ -122,6 +114,7 @@ export class WebsocketClient { stockId: string, tr_type: TR_IDS, ): string { + this.logger.info(JSON.stringify(config)); const message = { header: { approval_key: config.STOCK_WEBSOCKET_KEY!, @@ -139,12 +132,13 @@ export class WebsocketClient { return JSON.stringify(message); } - private sendMessage(message: string) { - if (this.client.readyState === WebSocket.OPEN) { - this.client.send(message); - this.logger.info(`Sent message: ${message}`); + private parseMessage(data: RawData) { + if (typeof data === 'object' && !(data instanceof Buffer)) { + return data; + } else if (typeof data === 'object') { + return parseMessage(data.toString()); } else { - this.logger.warn('WebSocket is not open. Message not sent.'); + return parseMessage(data as string); } } } diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 73f33f23..3f9b4aba 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -5,8 +5,9 @@ import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiTokenApi } from './api/openapiToken.api'; +import { LiveData } from './liveData.service'; import { OpenapiScraperService } from './openapi-scraper.service'; -import { WebsocketClient } from './websocketClient.service'; +import { WebsocketClient } from './websocket/websocketClient.websocket'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily, @@ -40,6 +41,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; OpenapiScraperService, OpenapiLiveData, WebsocketClient, + LiveData, ], exports: [WebsocketClient], }) diff --git a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts new file mode 100644 index 00000000..30bc93f3 --- /dev/null +++ b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts @@ -0,0 +1,59 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Inject, Injectable } from '@nestjs/common'; +import { Logger } from 'winston'; +import { RawData, WebSocket } from 'ws'; + +@Injectable() +export class WebsocketClient { + private readonly url = + process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; + private client: WebSocket = new WebSocket(this.url); + + constructor(@Inject('winston') private readonly logger: Logger) {} + + subscribe(message: string) { + this.sendMessage(message); + } + + discribe(message: string) { + this.sendMessage(message); + } + + // TODO : 분리 + private initDisconnect( + initCloseCallback: () => void, + initErrorCallback: (error: unknown) => void, + ) { + this.client.on('close', initCloseCallback); + + this.client.on('error', initErrorCallback); + } + + private initOpen(fn: () => void) { + this.client.on('open', fn); + } + + private initMessage(fn: (data: RawData) => void) { + this.client.on('message', fn); + } + + connect( + initOpenCallback: (fn: (message: string) => void) => () => void, + initMessageCallback: (client: WebSocket) => (data: RawData) => void, + initCloseCallback: () => void, + initErrorCallback: (error: unknown) => void, + ) { + this.initOpen(initOpenCallback(this.sendMessage)); + this.initMessage(initMessageCallback(this.client)); + this.initDisconnect(initCloseCallback, initErrorCallback); + } + + private sendMessage(message: string) { + if (this.client.readyState === WebSocket.OPEN) { + this.client.send(message); + this.logger.info(`Sent message: ${message}`); + } else { + this.logger.warn('WebSocket is not open. Message not sent.'); + } + } +} diff --git a/packages/backend/src/stock/stock.gateway.ts b/packages/backend/src/stock/stock.gateway.ts index c6268e82..9525d425 100644 --- a/packages/backend/src/stock/stock.gateway.ts +++ b/packages/backend/src/stock/stock.gateway.ts @@ -6,7 +6,7 @@ import { WebSocketServer, } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; -import { WebsocketClient } from '@/scraper/openapi/websocketClient.service'; +import { WebsocketClient } from '@/scraper/openapi/websocket/websocketClient.websocket'; @WebSocketGateway({ namespace: '/api/stock/realtime', From 6bcb5c5aa6d6df3d13a88083438724d645afa13d Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 18:06:52 +0900 Subject: [PATCH 102/112] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20liveda?= =?UTF-8?q?ta=20stock=20module=EB=A1=9C=20=EC=9D=B4=EB=8F=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/liveData.service.ts | 3 +-- .../backend/src/scraper/openapi/openapi-scraper.module.ts | 8 +------- packages/backend/src/stock/stock.gateway.ts | 6 +++--- packages/backend/src/stock/stock.module.ts | 8 ++++++++ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index a4825a18..20daa9c5 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -14,8 +14,6 @@ type TR_IDS = '0' | '1'; // TODO : 비즈니스 로직을 분리해야함. @Injectable() export class LiveData { - private readonly url = - process.env.WS_URL ?? 'ws://ops.koreainvestment.com:21000'; private readonly clientStock: Set = new Set(); private readonly reconnectInterval = 60 * 1000 * 1000; @@ -27,6 +25,7 @@ export class LiveData { ) { this.connect(); this.subscribe('000150'); + setTimeout(() => this.discribe('000150'), 15000); } async subscribe(stockId: string) { diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index 3f9b4aba..4afbf154 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -1,13 +1,10 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { OpenapiDetailData } from './api/openapiDetailData.api'; -import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; import { OpenapiTokenApi } from './api/openapiToken.api'; -import { LiveData } from './liveData.service'; import { OpenapiScraperService } from './openapi-scraper.service'; -import { WebsocketClient } from './websocket/websocketClient.websocket'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily, @@ -39,10 +36,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; OpenapiMinuteData, OpenapiDetailData, OpenapiScraperService, - OpenapiLiveData, - WebsocketClient, - LiveData, + OpenapiTokenApi, ], - exports: [WebsocketClient], }) export class OpenapiScraperModule {} diff --git a/packages/backend/src/stock/stock.gateway.ts b/packages/backend/src/stock/stock.gateway.ts index 9525d425..dd89423c 100644 --- a/packages/backend/src/stock/stock.gateway.ts +++ b/packages/backend/src/stock/stock.gateway.ts @@ -6,7 +6,7 @@ import { WebSocketServer, } from '@nestjs/websockets'; import { Server, Socket } from 'socket.io'; -import { WebsocketClient } from '@/scraper/openapi/websocket/websocketClient.websocket'; +import { LiveData } from '@/scraper/openapi/liveData.service'; @WebSocketGateway({ namespace: '/api/stock/realtime', @@ -17,7 +17,7 @@ export class StockGateway { @WebSocketServer() server: Server; - constructor(private readonly websocketClient: WebsocketClient) {} + constructor(private readonly liveData: LiveData) {} @SubscribeMessage('connectStock') async handleConnectStock( @@ -27,7 +27,7 @@ export class StockGateway { client.join(stockId); if ((await this.server.in(stockId).fetchSockets()).length === 0) { - this.websocketClient.subscribe(stockId); + this.liveData.subscribe(stockId); } client.emit('connectionSuccess', { message: `Successfully connected to stock room: ${stockId}`, diff --git a/packages/backend/src/stock/stock.module.ts b/packages/backend/src/stock/stock.module.ts index 69d925f4..11b81eca 100644 --- a/packages/backend/src/stock/stock.module.ts +++ b/packages/backend/src/stock/stock.module.ts @@ -23,7 +23,11 @@ import { } from './stockData.service'; import { StockDetailService } from './stockDetail.service'; import { StockLiveDataSubscriber } from './stockLiveData.subscriber'; +import { OpenapiLiveData } from '@/scraper/openapi/api/openapiLiveData.api'; +import { OpenapiTokenApi } from '@/scraper/openapi/api/openapiToken.api'; +import { LiveData } from '@/scraper/openapi/liveData.service'; import { OpenapiScraperModule } from '@/scraper/openapi/openapi-scraper.module'; +import { WebsocketClient } from '@/scraper/openapi/websocket/websocketClient.websocket'; @Module({ imports: [ @@ -42,6 +46,10 @@ import { OpenapiScraperModule } from '@/scraper/openapi/openapi-scraper.module'; controllers: [StockController], providers: [ StockService, + WebsocketClient, + OpenapiTokenApi, + OpenapiLiveData, + LiveData, StockGateway, StockLiveDataSubscriber, StockDataService, From 77e410535a898336d0ab6d5c23ec330c1eb38e65 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 19:30:26 +0900 Subject: [PATCH 103/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EC=9E=A5=20?= =?UTF-8?q?=EB=A7=88=EA=B0=90=EC=8B=9C=20openapi=EB=A1=9C=20=EB=B6=80?= =?UTF-8?q?=EB=A5=B4=EB=8A=94=20=EB=A1=9C=EC=A7=81=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/openapi/liveData.service.ts | 37 ++++++++++++++----- .../websocket/websocketClient.websocket.ts | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 20daa9c5..82e1c574 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -11,12 +11,14 @@ import { parseMessage } from './parse/openapi.parser'; import { WebsocketClient } from './websocket/websocketClient.websocket'; type TR_IDS = '0' | '1'; -// TODO : 비즈니스 로직을 분리해야함. + @Injectable() export class LiveData { private readonly clientStock: Set = new Set(); private readonly reconnectInterval = 60 * 1000 * 1000; + private readonly startTime: Date = new Date(2024, 0, 1, 9, 0, 0, 0); + private readonly endTime: Date = new Date(2024, 0, 1, 15, 30, 0, 0); constructor( private readonly openApiToken: OpenapiTokenApi, private readonly webSocketClient: WebsocketClient, @@ -29,14 +31,21 @@ export class LiveData { } async subscribe(stockId: string) { - this.clientStock.add(stockId); - // TODO : 하나의 config만 사용중. - const message = this.convertObjectToMessage( - (await this.openApiToken.configs())[0], - stockId, - '1', - ); - this.webSocketClient.subscribe(message); + if (this.isCloseTime(new Date(), this.startTime, this.endTime)) { + const result = await this.openapiLiveData.connectLiveData(stockId); + const stockLiveData = this.openapiLiveData.convertLiveData(result); + this.logger.info('in open api'); + this.openapiLiveData.saveLiveData(stockLiveData); + } else { + this.clientStock.add(stockId); + // TODO : 하나의 config만 사용중. + const message = this.convertObjectToMessage( + (await this.openApiToken.configs())[0], + stockId, + '1', + ); + this.webSocketClient.subscribe(message); + } } async discribe(stockId: string) { @@ -98,9 +107,17 @@ export class LiveData { setTimeout(() => this.connect(), this.reconnectInterval); }; + private isCloseTime(date: Date, start: Date, end: Date): boolean { + const dateMinutes = date.getHours() * 60 + date.getMinutes(); + const startMinutes = start.getHours() * 60 + start.getMinutes(); + const endMinutes = end.getHours() * 60 + end.getMinutes(); + + return dateMinutes >= startMinutes && dateMinutes <= endMinutes; + } + @Cron('0 2 * * 1-5') connect() { - this.webSocketClient.connect( + this.webSocketClient.connectPacade( this.initOpenCallback, this.initMessageCallback, this.initCloseCallback, diff --git a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts index 30bc93f3..f9e4a5e3 100644 --- a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts +++ b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts @@ -37,7 +37,7 @@ export class WebsocketClient { this.client.on('message', fn); } - connect( + connectPacade( initOpenCallback: (fn: (message: string) => void) => () => void, initMessageCallback: (client: WebSocket) => (data: RawData) => void, initCloseCallback: () => void, From e82a6dd61e60d917ed8ac859bd335226a3f51dc7 Mon Sep 17 00:00:00 2001 From: sunghwki <52474291+swkim12345@users.noreply.github.com> Date: Mon, 25 Nov 2024 19:35:21 +0900 Subject: [PATCH 104/112] =?UTF-8?q?Bug/#235=20websocket=20=EB=B2=84?= =?UTF-8?q?=EA=B7=B8=20=ED=95=B4=EA=B2=B0,=20token=20db=EC=97=90=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5=20(#240)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat: token entity 추가 * ✨ feat: entity 에 저장, expire 검사 로직 추가 * 🐛 fix: token 주입으로 로직 변경 * ♻️ refactor: token 주입으로 변경, 그로 인한 오류 수정 및 console.log 삭제 * 🐛 fix: live 데이터 수집 오류 해결, 데이터 없을 때 insert 오류 해결 * 🐛 fix: stock, livedata entity 수정 * ♻️ refactor: develop 환경시 logging 활성화 * 📦️ ci: production 환경일 때 작동되게 변경 --- packages/backend/src/configs/typeormConfig.ts | 2 +- .../src/scraper/domain/openapiToken.entity.ts | 28 ++++ .../openapi/api/openapiDetailData.api.ts | 9 +- .../openapi/api/openapiLiveData.api.ts | 39 ++++-- .../openapi/api/openapiMinuteData.api.ts | 11 +- .../openapi/api/openapiPeriodData.api.ts | 9 +- .../scraper/openapi/api/openapiToken.api.ts | 125 ++++++++++++++++-- .../scraper/openapi/openapi-scraper.module.ts | 2 + .../src/scraper/openapi/util/priorityQueue.ts | 13 -- .../openapi/websocketClient.service.ts | 32 ++--- .../backend/src/stock/domain/stock.entity.ts | 4 + 11 files changed, 210 insertions(+), 64 deletions(-) create mode 100644 packages/backend/src/scraper/domain/openapiToken.entity.ts diff --git a/packages/backend/src/configs/typeormConfig.ts b/packages/backend/src/configs/typeormConfig.ts index a8c70a18..f8ff59ac 100644 --- a/packages/backend/src/configs/typeormConfig.ts +++ b/packages/backend/src/configs/typeormConfig.ts @@ -21,6 +21,6 @@ export const typeormDevelopConfig: TypeOrmModuleOptions = { password: process.env.DB_PASS, database: process.env.DB_NAME, entities: [__dirname + '/../**/*.entity.{js,ts}'], - //logging: true, + logging: true, synchronize: true, }; diff --git a/packages/backend/src/scraper/domain/openapiToken.entity.ts b/packages/backend/src/scraper/domain/openapiToken.entity.ts new file mode 100644 index 00000000..613abcb5 --- /dev/null +++ b/packages/backend/src/scraper/domain/openapiToken.entity.ts @@ -0,0 +1,28 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'openapi_token' }) +export class OpenapiToken { + @PrimaryColumn({ name: 'account' }) + account: string; + + @Column({ name: 'apiUrl' }) + api_url: string; + + @Column({ name: 'key' }) + api_key: string; + + @Column({ name: 'password' }) + api_password: string; + + @Column({ name: 'token', length: 512 }) + api_token?: string; + + @Column({ name: 'tokenExpire' }) + api_token_expire?: Date; + + @Column({ name: 'websocketKey' }) + websocket_key?: string; + + @Column({ name: 'websocketKeyExpire' }) + websocket_key_expire?: Date; +} diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 9be8e3ea..6a32435f 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -13,7 +13,7 @@ import { } from '../type/openapiDetailData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { KospiStock } from '@/stock/domain/kospiStock.entity'; import { Stock } from '@/stock/domain/stock.entity'; import { StockDaily } from '@/stock/domain/stockData.entity'; @@ -27,6 +27,7 @@ export class OpenapiDetailData { '/uapi/domestic-stock/v1/quotations/search-stock-info'; private readonly intervals = 1000; constructor( + private readonly openApiToken: OpenapiTokenApi, private readonly datasource: DataSource, @Inject('winston') private readonly logger: Logger, ) { @@ -38,13 +39,13 @@ export class OpenapiDetailData { if (process.env.NODE_ENV !== 'production') return; const entityManager = this.datasource.manager; const stocks = await entityManager.find(Stock); - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { - this.logger.info(openApiToken.configs[i]); + this.logger.info(this.openApiToken.configs[i]); const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize); - this.getDetailDataChunk(chunk, openApiToken.configs[i]); + this.getDetailDataChunk(chunk, this.openApiToken.configs[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 16976657..6fa15505 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -1,26 +1,47 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; +import { Logger } from 'winston'; +import { Stock } from '@/stock/domain/stock.entity'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @Injectable() export class OpenapiLiveData { public readonly TR_ID: string = 'H0STCNT0'; - constructor(private readonly datasource: DataSource) {} + constructor( + private readonly datasource: DataSource, + @Inject('winston') private readonly logger: Logger, + ) {} async saveLiveData(data: StockLiveData[]) { - await this.datasource.manager - .getRepository(StockLiveData) - .createQueryBuilder() - .insert() - .into(StockLiveData) - .values(data) - .execute(); + const exists = await this.datasource.manager.exists(StockLiveData, { + where: { + stock: { id: data[0].stock.id }, + }, + }); + if (exists) { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .update() + .set(data[0]) + .where('stock.id = :stockId', { stockId: data[0].stock.id }) + .execute(); + } else { + await this.datasource.manager + .getRepository(StockLiveData) + .createQueryBuilder() + .insert() + .into(StockLiveData) + .values(data) + .execute(); + } } convertLiveData(messages: Record[]): StockLiveData[] { const stockData: StockLiveData[] = []; messages.map((message) => { const stockLiveData = new StockLiveData(); + stockLiveData.stock = { id: message.STOCK_ID } as Stock; stockLiveData.currentPrice = parseFloat(message.STCK_PRPR); stockLiveData.changeRate = parseFloat(message.PRDY_CTRT); stockLiveData.volume = parseInt(message.CNTG_VOL); diff --git a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts index 003e7560..e64e26c7 100644 --- a/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiMinuteData.api.ts @@ -11,7 +11,7 @@ import { } from '../type/openapiMinuteData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, StockMinutely } from '@/stock/domain/stockData.entity'; @@ -27,9 +27,10 @@ export class OpenapiMinuteData { private flip: number = 0; constructor( private readonly datasource: DataSource, + private readonly openApiToken: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) { - this.getStockData(); + //this.getStockData(); } @Cron('0 1 * * 1-5') @@ -126,16 +127,16 @@ export class OpenapiMinuteData { } } - @Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) + //@Cron(`*/${STOCK_CUT} 9-15 * * 1-5`) getMinuteData() { if (process.env.NODE_ENV !== 'production') return; - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const stock = this.stock[this.flip % STOCK_CUT]; this.flip++; const chunkSize = Math.ceil(stock.length / configCount); for (let i = 0; i < configCount; i++) { const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize); - this.getMinuteDataChunk(chunk, openApiToken.configs[i]); + this.getMinuteDataChunk(chunk, this.openApiToken.configs[i]); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index f4268088..4b976be8 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -14,7 +14,7 @@ import { getPreviousDate, getTodayDate, } from '../util/openapiUtil.api'; -import { openApiToken } from './openapiToken.api'; +import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockData, @@ -46,6 +46,7 @@ export class OpenapiPeriodData { '/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice'; constructor( private readonly datasource: DataSource, + private readonly openApiToken: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) { //this.getItemChartPriceCheck(); @@ -59,7 +60,7 @@ export class OpenapiPeriodData { isTrading: true, }, }); - const configCount = openApiToken.configs.length; + const configCount = this.openApiToken.configs.length; const chunkSize = Math.ceil(stocks.length / configCount); for (let i = 0; i < configCount; i++) { @@ -95,7 +96,7 @@ export class OpenapiPeriodData { let isFail = false; while (!isFail) { - configIdx = (configIdx + 1) % openApiToken.configs.length; + configIdx = (configIdx + 1) % this.openApiToken.configs.length; this.setStockPeriod(stockPeriod, stock.id!, end); // chart 데이터가 있는 지 확인 -> 리턴 @@ -129,7 +130,7 @@ export class OpenapiPeriodData { try { const response = await getOpenApi( this.url, - openApiToken.configs[configIdx], + this.openApiToken.configs[configIdx], query, TR_IDS.ITEM_CHART_PRICE, ); diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index 6e6c88c5..829ca9a3 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -1,14 +1,21 @@ -import { Inject } from '@nestjs/common'; +import { Global, Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; +import { DataSource } from 'typeorm'; import { Logger } from 'winston'; +import { OpenapiToken } from '../../domain/openapiToken.entity'; import { openApiConfig } from '../config/openapi.config'; import { OpenapiException } from '../util/openapiCustom.error'; import { postOpenApi } from '../util/openapiUtil.api'; -import { logger } from '@/configs/logger.config'; -class OpenapiTokenApi { +@Global() +@Injectable() +export class OpenapiTokenApi { private config: (typeof openApiConfig)[] = []; - constructor(@Inject('winston') private readonly logger: Logger) { + constructor( + @Inject('winston') private readonly logger: Logger, + private readonly datasource: DataSource, + ) { + if (process.env.NODE_ENV !== 'production') return; const accounts = openApiConfig.STOCK_ACCOUNT!.split(','); const api_keys = openApiConfig.STOCK_API_KEY!.split(','); const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(','); @@ -27,14 +34,111 @@ class OpenapiTokenApi { STOCK_API_PASSWORD: api_passwords[i], }); } - this.initAuthenValue(); + this.init(); } get configs() { - //TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경 + this.init(); return this.config; } + @Cron('30 0 * * 1-5') + async init() { + const tokens = await this.convertConfigToTokenEntity(this.config); + const config = await this.getPropertyFromDB(tokens); + const expired = config.filter( + (val) => + this.isTokenExpired(val.api_token_expire) && + this.isTokenExpired(val.websocket_key_expire), + ); + + if (expired.length || !config.length) { + await this.initAuthenValue(); + const newTokens = await this.convertConfigToTokenEntity(this.config); + this.savePropertyToDB(newTokens); + } else { + this.config = await this.convertTokenEntityToConfig(config); + } + } + + private isTokenExpired(startDate?: Date) { + if (!startDate) return true; + const now = new Date(); + //실제 만료 시간은 24시간이지만, 문제의 소지가 발생하는 것을 방지하기 위해 20시간으로 설정함. + const baseTimeToMilliSec = 20 * 60 * 60 * 1000; + const timeDiff = now.getTime() - startDate.getTime(); + + return timeDiff >= baseTimeToMilliSec; + } + + private async convertTokenEntityToConfig(tokens: OpenapiToken[]) { + const result: (typeof openApiConfig)[] = []; + tokens.forEach((val) => { + const config: typeof openApiConfig = { + STOCK_ACCOUNT: val.account, + STOCK_API_KEY: val.api_key, + STOCK_API_PASSWORD: val.api_password, + STOCK_API_TOKEN: val.api_token, + STOCK_URL: val.api_url, + STOCK_WEBSOCKET_KEY: val.websocket_key, + }; + result.push(config); + }); + return result; + } + + private async convertConfigToTokenEntity(config: (typeof openApiConfig)[]) { + const result: OpenapiToken[] = []; + config.forEach((val) => { + const token = new OpenapiToken(); + if ( + val.STOCK_URL && + val.STOCK_ACCOUNT && + val.STOCK_API_KEY && + val.STOCK_API_PASSWORD + ) { + token.api_url = val.STOCK_URL; + token.account = val.STOCK_ACCOUNT; + token.api_key = val.STOCK_API_KEY; + token.api_password = val.STOCK_API_PASSWORD; + } + token.api_token = val.STOCK_API_TOKEN; + token.websocket_key = val.STOCK_WEBSOCKET_KEY; + token.api_token_expire = new Date(); + token.websocket_key_expire = new Date(); + result.push(token); + }); + return result; + } + + private async savePropertyToDB(tokens: OpenapiToken[]) { + tokens.forEach(async (val) => { + this.datasource.manager.save(OpenapiToken, val); + }); + } + + private async getPropertyFromDB(tokens: OpenapiToken[]) { + const result: OpenapiToken[] = []; + await Promise.all( + tokens.map(async (val) => { + const findByToken = await this.datasource.manager.findOne( + OpenapiToken, + { + where: { + account: val.account, + api_key: val.api_key, + api_password: val.api_password, + }, + }, + ); + if (findByToken) { + result.push(findByToken); + } + }), + ); + return result; + } + private async initAuthenValue() { const delay = 60000; const delayMinute = delay / 1000 / 60; @@ -59,8 +163,7 @@ class OpenapiTokenApi { } } - @Cron('50 0 * * 1-5') - async initAccessToken() { + private async initAccessToken() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_API_TOKEN = await this.getToken(val)!; @@ -70,8 +173,7 @@ class OpenapiTokenApi { this.config = updatedConfig; } - @Cron('50 0 * * 1-5') - async initWebSocketKey() { + private async initWebSocketKey() { const updatedConfig = await Promise.all( this.config.map(async (val) => { val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!; @@ -107,6 +209,3 @@ class OpenapiTokenApi { return tmp.approval_key as string; } } - -const openApiToken = new OpenapiTokenApi(logger); -export { openApiToken }; diff --git a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts index ad425412..73f33f23 100644 --- a/packages/backend/src/scraper/openapi/openapi-scraper.module.ts +++ b/packages/backend/src/scraper/openapi/openapi-scraper.module.ts @@ -4,6 +4,7 @@ import { OpenapiDetailData } from './api/openapiDetailData.api'; import { OpenapiLiveData } from './api/openapiLiveData.api'; import { OpenapiMinuteData } from './api/openapiMinuteData.api'; import { OpenapiPeriodData } from './api/openapiPeriodData.api'; +import { OpenapiTokenApi } from './api/openapiToken.api'; import { OpenapiScraperService } from './openapi-scraper.service'; import { WebsocketClient } from './websocketClient.service'; import { Stock } from '@/stock/domain/stock.entity'; @@ -32,6 +33,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; ], controllers: [], providers: [ + OpenapiTokenApi, OpenapiPeriodData, OpenapiMinuteData, OpenapiDetailData, diff --git a/packages/backend/src/scraper/openapi/util/priorityQueue.ts b/packages/backend/src/scraper/openapi/util/priorityQueue.ts index a49e5822..190718a5 100644 --- a/packages/backend/src/scraper/openapi/util/priorityQueue.ts +++ b/packages/backend/src/scraper/openapi/util/priorityQueue.ts @@ -84,16 +84,3 @@ export class PriorityQueue { return this.heap.length === 0; } } - -const pq = new PriorityQueue(); - -pq.enqueue('Task A', 2); -pq.enqueue('Task B', 1); -pq.enqueue('Task C', 3); - -console.log(pq.dequeue()); // Task B -console.log(pq.peek()); // Task A -console.log(pq.dequeue()); // Task A -console.log(pq.isEmpty()); // false -console.log(pq.dequeue()); // Task C -console.log(pq.isEmpty()); // true diff --git a/packages/backend/src/scraper/openapi/websocketClient.service.ts b/packages/backend/src/scraper/openapi/websocketClient.service.ts index 40501a25..e784afe5 100644 --- a/packages/backend/src/scraper/openapi/websocketClient.service.ts +++ b/packages/backend/src/scraper/openapi/websocketClient.service.ts @@ -2,9 +2,9 @@ import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; -import { WebSocket } from 'ws'; +import { RawData, WebSocket } from 'ws'; import { OpenapiLiveData } from './api/openapiLiveData.api'; -import { openApiToken } from './api/openapiToken.api'; +import { OpenapiTokenApi } from './api/openapiToken.api'; import { openApiConfig } from './config/openapi.config'; import { parseMessage } from './parse/openapi.parser'; @@ -20,6 +20,7 @@ export class WebsocketClient { constructor( @Inject('winston') private readonly logger: Logger, + private readonly openApiToken: OpenapiTokenApi, private readonly openapiLiveData: OpenapiLiveData, ) { if (process.env.NODE_ENV === 'production') { @@ -32,7 +33,7 @@ export class WebsocketClient { this.clientStock.add(stockId); // TODO : 하나의 config만 사용중. const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '1', ); @@ -42,7 +43,7 @@ export class WebsocketClient { discribe(stockId: string) { this.clientStock.delete(stockId); const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '0', ); @@ -67,7 +68,7 @@ export class WebsocketClient { this.logger.info('WebSocket connection established'); for (const stockId of this.clientStock.keys()) { const message = this.convertObjectToMessage( - openApiToken.configs[0], + this.openApiToken.configs[0], stockId, '1', ); @@ -77,31 +78,32 @@ export class WebsocketClient { } private initMessage() { - this.client.on('message', async (data) => { + this.client.on('message', async (data: RawData) => { try { - const message = this.parseMessage(data.toString()); + console.log(data); + const message = this.parseMessage(data); if (message.header) { if (message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${JSON.stringify(data)}`); - this.client.pong({ - tr_id: 'PINGPONG', - datetime: new Date().toISOString(), - }); + this.logger.info(`Received PING: ${data}`); + this.client.pong(data); } return; } this.logger.info(`Recived data : ${data}`); + this.logger.info(`Stock id : ${message[0]['STOCK_ID']}`); const liveData = this.openapiLiveData.convertLiveData(message); - this.openapiLiveData.saveLiveData(liveData); + await this.openapiLiveData.saveLiveData(liveData); } catch (error) { this.logger.warn(error); } }); } - private parseMessage(data: string) { - if (typeof data === 'object') { + private parseMessage(data: RawData) { + if (typeof data === 'object' && !(data instanceof Buffer)) { return data; + } else if (typeof data === 'object') { + return parseMessage(data.toString()); } else { return parseMessage(data as string); } diff --git a/packages/backend/src/stock/domain/stock.entity.ts b/packages/backend/src/stock/domain/stock.entity.ts index be321f55..994a6de1 100644 --- a/packages/backend/src/stock/domain/stock.entity.ts +++ b/packages/backend/src/stock/domain/stock.entity.ts @@ -7,6 +7,7 @@ import { StockWeekly, StockYearly, } from './stockData.entity'; +import { StockLiveData } from './stockLiveData.entity'; import { Like } from '@/chat/domain/like.entity'; import { DateEmbedded } from '@/common/dateEmbedded.entity'; import { UserStock } from '@/stock/domain/userStock.entity'; @@ -52,6 +53,9 @@ export class Stock { @OneToMany(() => StockYearly, (stockYearly) => stockYearly.stock) stockYearly?: StockYearly[]; + @OneToOne(() => StockLiveData, (stockLiveData) => stockLiveData.stock) + stockLive?: StockLiveData; + @OneToOne(() => KospiStock, (kospiStock) => kospiStock.stock) kospiStock?: KospiStock; } From 44bb8eceb6d2423492ad738abcb4542932fb136f Mon Sep 17 00:00:00 2001 From: kimminsu Date: Mon, 25 Nov 2024 19:39:40 +0900 Subject: [PATCH 105/112] =?UTF-8?q?=E2=9C=A8=20feat:=20=EA=B1=B0=EB=9E=98?= =?UTF-8?q?=EC=A4=91=EC=9D=B8=20=EC=A2=85=EB=AA=A9=EB=A7=8C=20=EA=B2=80?= =?UTF-8?q?=EC=83=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/stock.service.ts | 61 ++++++++++----------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/stock/stock.service.ts b/packages/backend/src/stock/stock.service.ts index 154eb9d6..c0e50a9c 100644 --- a/packages/backend/src/stock/stock.service.ts +++ b/packages/backend/src/stock/stock.service.ts @@ -69,11 +69,10 @@ export class StockService { } async searchStock(stockName: string) { - const queryBuilder = this.datasource + const result = await this.datasource .getRepository(Stock) - .createQueryBuilder(); - const result = await queryBuilder - .where('stock.stock_name LIKE :name', { + .createQueryBuilder('stock') + .where('stock.is_trading = :isTrading and stock.stock_name LIKE :name', { isTrading: true, name: `%${stockName}%`, }) @@ -94,6 +93,33 @@ export class StockService { } } + async getTopStocksByViews(limit: number) { + const rawData = await this.StocksQuery() + .orderBy('stock.views', 'DESC') + .limit(limit) + .getRawMany(); + + return plainToInstance(StocksResponse, rawData); + } + + async getTopStocksByGainers(limit: number) { + const rawData = await this.StocksQuery() + .orderBy('stockLiveData.changeRate', 'DESC') + .limit(limit) + .getRawMany(); + + return plainToInstance(StocksResponse, rawData); + } + + async getTopStocksByLosers(limit: number) { + const rawData = await this.StocksQuery() + .orderBy('stockLiveData.changeRate', 'ASC') + .limit(limit) + .getRawMany(); + + return plainToInstance(StocksResponse, rawData); + } + private async validateStockExists(stockId: string, manager: EntityManager) { if (!(await this.existsStock(stockId, manager))) { throw new BadRequestException('not exists stock'); @@ -150,31 +176,4 @@ export class StockService { 'stockDetail.marketCap AS marketCap', ]); } - - async getTopStocksByViews(limit: number) { - const rawData = await this.StocksQuery() - .orderBy('stock.views', 'DESC') - .limit(limit) - .getRawMany(); - - return plainToInstance(StocksResponse, rawData); - } - - async getTopStocksByGainers(limit: number) { - const rawData = await this.StocksQuery() - .orderBy('stockLiveData.changeRate', 'DESC') - .limit(limit) - .getRawMany(); - - return plainToInstance(StocksResponse, rawData); - } - - async getTopStocksByLosers(limit: number) { - const rawData = await this.StocksQuery() - .orderBy('stockLiveData.changeRate', 'ASC') - .limit(limit) - .getRawMany(); - - return plainToInstance(StocksResponse, rawData); - } } From fe29d5014760da6fe34d53b763ff88f5e51d5a50 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 19:49:45 +0900 Subject: [PATCH 106/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20websocket=20logge?= =?UTF-8?q?r=20=EC=B6=94=EA=B0=80,=20client=20stock=20=EC=A0=80=EC=9E=A5?= =?UTF-8?q?=EB=90=98=EC=A7=80=20=EC=95=8A=EB=8A=94=20=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/liveData.service.ts | 6 +----- .../scraper/openapi/websocket/websocketClient.websocket.ts | 2 ++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 82e1c574..0c69f5e9 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -1,5 +1,3 @@ -//TODO : 9시 ~ 3시 반까지는 openapi에서 가져오고, 아니면 websocket으로 가져오기 - import { Inject, Injectable } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { Logger } from 'winston'; @@ -26,18 +24,16 @@ export class LiveData { @Inject('winston') private readonly logger: Logger, ) { this.connect(); - this.subscribe('000150'); - setTimeout(() => this.discribe('000150'), 15000); } async subscribe(stockId: string) { + this.clientStock.add(stockId); if (this.isCloseTime(new Date(), this.startTime, this.endTime)) { const result = await this.openapiLiveData.connectLiveData(stockId); const stockLiveData = this.openapiLiveData.convertLiveData(result); this.logger.info('in open api'); this.openapiLiveData.saveLiveData(stockLiveData); } else { - this.clientStock.add(stockId); // TODO : 하나의 config만 사용중. const message = this.convertObjectToMessage( (await this.openApiToken.configs())[0], diff --git a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts index f9e4a5e3..06451a53 100644 --- a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts +++ b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts @@ -12,10 +12,12 @@ export class WebsocketClient { constructor(@Inject('winston') private readonly logger: Logger) {} subscribe(message: string) { + this.logger.info(`Subscribe : ${message}`); this.sendMessage(message); } discribe(message: string) { + this.logger.info(`Discribe : ${message}`); this.sendMessage(message); } From 007e2e52412947047690000fa70936c6ada38e69 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 19:51:29 +0900 Subject: [PATCH 107/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openapi=EB=8A=94?= =?UTF-8?q?=20=EC=A0=80=EC=9E=A5=EC=9D=B4=20=ED=95=84=EC=9A=94=20=EC=97=86?= =?UTF-8?q?=EC=9D=8C=20=EB=A1=A4=20=EB=B0=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/openapi/liveData.service.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 0c69f5e9..7230af30 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -27,7 +27,6 @@ export class LiveData { } async subscribe(stockId: string) { - this.clientStock.add(stockId); if (this.isCloseTime(new Date(), this.startTime, this.endTime)) { const result = await this.openapiLiveData.connectLiveData(stockId); const stockLiveData = this.openapiLiveData.convertLiveData(result); @@ -35,6 +34,7 @@ export class LiveData { this.openapiLiveData.saveLiveData(stockLiveData); } else { // TODO : 하나의 config만 사용중. + this.clientStock.add(stockId); const message = this.convertObjectToMessage( (await this.openApiToken.configs())[0], stockId, @@ -45,13 +45,15 @@ export class LiveData { } async discribe(stockId: string) { - this.clientStock.delete(stockId); - const message = this.convertObjectToMessage( - (await this.openApiToken.configs())[0], - stockId, - '0', - ); - this.webSocketClient.discribe(message); + if (this.clientStock.has(stockId)) { + this.clientStock.delete(stockId); + const message = this.convertObjectToMessage( + (await this.openApiToken.configs())[0], + stockId, + '0', + ); + this.webSocketClient.discribe(message); + } } private initOpenCallback = From ff9ad022b653f79ac89fbdf4f0a0dfef8fcc8326 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 22:45:15 +0900 Subject: [PATCH 108/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20open=20api?= =?UTF-8?q?=EB=A1=9C=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EB=B0=9B=EC=A7=80=20?= =?UTF-8?q?=EB=AA=BB=ED=95=98=EB=8A=94=20=EB=AC=B8=EC=A0=9C=20=ED=95=B4?= =?UTF-8?q?=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/configs/typeormConfig.ts | 2 +- .../openapi/api/openapiLiveData.api.ts | 29 ++++- .../scraper/openapi/api/openapiToken.api.ts | 17 +-- .../src/scraper/openapi/liveData.service.ts | 25 ++++- .../openapi/type/openapiLiveData.type.ts | 105 ++++++++++++++++++ .../scraper/openapi/type/openapiUtil.type.ts | 2 + .../src/stock/domain/stockLiveData.entity.ts | 3 - 7 files changed, 162 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/configs/typeormConfig.ts b/packages/backend/src/configs/typeormConfig.ts index f8ff59ac..a8c70a18 100644 --- a/packages/backend/src/configs/typeormConfig.ts +++ b/packages/backend/src/configs/typeormConfig.ts @@ -21,6 +21,6 @@ export const typeormDevelopConfig: TypeOrmModuleOptions = { password: process.env.DB_PASS, database: process.env.DB_NAME, entities: [__dirname + '/../**/*.entity.{js,ts}'], - logging: true, + //logging: true, synchronize: true, }; diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 27375f83..6ffccb8a 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -1,6 +1,8 @@ import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { Logger } from 'winston'; +import { openApiConfig } from '../config/openapi.config'; +import { isOpenapiLiveData } from '../type/openapiLiveData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getOpenApi } from '../util/openapiUtil.api'; import { OpenapiTokenApi } from './openapiToken.api'; @@ -42,6 +44,25 @@ export class OpenapiLiveData { } } + convertResponseToStockLiveData = ( + data: OpenapiLiveData, + stockId: string, + ): StockLiveData | undefined => { + const stockLiveData = new StockLiveData(); + if (isOpenapiLiveData(data)) { + stockLiveData.stock = { id: stockId } as Stock; + stockLiveData.currentPrice = parseFloat(data.stck_prpr); + stockLiveData.changeRate = parseFloat(data.prdy_ctrt); + stockLiveData.volume = parseInt(data.acml_vol); + stockLiveData.high = parseFloat(data.stck_hgpr); + stockLiveData.low = parseFloat(data.stck_lwpr); + stockLiveData.open = parseFloat(data.stck_oprc); + stockLiveData.updatedAt = new Date(); + + return stockLiveData; + } + }; + convertLiveData(messages: Record[]): StockLiveData[] { const stockData: StockLiveData[] = []; messages.map((message) => { @@ -53,22 +74,22 @@ export class OpenapiLiveData { stockLiveData.high = parseFloat(message.STCK_HGPR); stockLiveData.low = parseFloat(message.STCK_LWPR); stockLiveData.open = parseFloat(message.STCK_OPRC); - stockLiveData.previousClose = parseFloat(message.WGHN_AVRG_STCK_PRC); stockLiveData.updatedAt = new Date(); + stockData.push(stockLiveData); }); return stockData; } - async connectLiveData(stockId: string) { + async connectLiveData(stockId: string, config: typeof openApiConfig) { const query = this.makeLiveDataQuery(stockId); try { const result = await getOpenApi( this.url, - (await this.config.configs())[0], + config, query, - TR_IDS.ITEM_CHART_PRICE, + TR_IDS.LIVE_DATA, ); return result; } catch (error) { diff --git a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts index deedce9b..fee72534 100644 --- a/packages/backend/src/scraper/openapi/api/openapiToken.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiToken.api.ts @@ -33,7 +33,6 @@ export class OpenapiTokenApi { STOCK_API_PASSWORD: api_passwords[i], }); } - this.init(); } async configs() { @@ -43,7 +42,7 @@ export class OpenapiTokenApi { @Cron('30 0 * * 1-5') async init() { - const tokens = await this.convertConfigToTokenEntity(this.config); + const tokens = this.convertConfigToTokenEntity(this.config); const config = await this.getPropertyFromDB(tokens); const expired = config.filter( (val) => @@ -53,24 +52,24 @@ export class OpenapiTokenApi { if (expired.length || !config.length) { await this.initAuthenValue(); - const newTokens = await this.convertConfigToTokenEntity(this.config); - this.savePropertyToDB(newTokens); + const newTokens = this.convertConfigToTokenEntity(this.config); + await this.savePropertyToDB(newTokens); } else { - this.config = await this.convertTokenEntityToConfig(config); + this.config = this.convertTokenEntityToConfig(config); } } private isTokenExpired(startDate?: Date) { if (!startDate) return true; const now = new Date(); - //실제 만료 시간은 24시간이지만, 문제의 소지가 발생하는 것을 방지하기 위해 20시간으로 설정함. + //실제 만료 시간은 24시간이지만, 문제가 발생할 여지를 줄이기 위해 20시간으로 설정 const baseTimeToMilliSec = 20 * 60 * 60 * 1000; const timeDiff = now.getTime() - startDate.getTime(); return timeDiff >= baseTimeToMilliSec; } - private async convertTokenEntityToConfig(tokens: OpenapiToken[]) { + private convertTokenEntityToConfig(tokens: OpenapiToken[]) { const result: (typeof openApiConfig)[] = []; tokens.forEach((val) => { const config: typeof openApiConfig = { @@ -86,7 +85,7 @@ export class OpenapiTokenApi { return result; } - private async convertConfigToTokenEntity(config: (typeof openApiConfig)[]) { + private convertConfigToTokenEntity(config: (typeof openApiConfig)[]) { const result: OpenapiToken[] = []; config.forEach((val) => { const token = new OpenapiToken(); @@ -170,6 +169,7 @@ export class OpenapiTokenApi { }), ); this.config = updatedConfig; + this.logger.info(`Init access token : ${this.config}`); } private async initWebSocketKey() { @@ -180,6 +180,7 @@ export class OpenapiTokenApi { }), ); this.config = updatedConfig; + this.logger.info(`Init websocket token : ${this.config}`); } private async getToken(config: typeof openApiConfig): Promise { diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 7230af30..343540e2 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -24,14 +24,29 @@ export class LiveData { @Inject('winston') private readonly logger: Logger, ) { this.connect(); + this.subscribe('000020'); + } + + private async openapiSubscribe(stockId: string) { + const config = (await this.openApiToken.configs())[0]; + const result = await this.openapiLiveData.connectLiveData(stockId, config); + this.logger.info(JSON.stringify(result)); + try { + const stockLiveData = this.openapiLiveData.convertResponseToStockLiveData( + result.output, + stockId, + ); + if (stockLiveData) { + this.openapiLiveData.saveLiveData([stockLiveData]); + } + } catch (error) { + this.logger.warn(`Subscribe error in open api : ${error}`); + } } async subscribe(stockId: string) { if (this.isCloseTime(new Date(), this.startTime, this.endTime)) { - const result = await this.openapiLiveData.connectLiveData(stockId); - const stockLiveData = this.openapiLiveData.convertLiveData(result); - this.logger.info('in open api'); - this.openapiLiveData.saveLiveData(stockLiveData); + await this.openapiSubscribe(stockId); } else { // TODO : 하나의 config만 사용중. this.clientStock.add(stockId); @@ -110,7 +125,7 @@ export class LiveData { const startMinutes = start.getHours() * 60 + start.getMinutes(); const endMinutes = end.getHours() * 60 + end.getMinutes(); - return dateMinutes >= startMinutes && dateMinutes <= endMinutes; + return dateMinutes <= startMinutes || dateMinutes >= endMinutes; } @Cron('0 2 * * 1-5') diff --git a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts index e1687cee..18bead3e 100644 --- a/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiLiveData.type.ts @@ -148,3 +148,108 @@ export const stockDataKeys = [ 'MRKT_TRTM_CLS_CODE', 'VI_STND_PRC', ]; + +export type OpenapiLiveData = { + iscd_stat_cls_code: string; + marg_rate: string; + rprs_mrkt_kor_name: string; + bstp_kor_isnm: string; + temp_stop_yn: string; + oprc_rang_cont_yn: string; + clpr_rang_cont_yn: string; + crdt_able_yn: string; + grmn_rate_cls_code: string; + elw_pblc_yn: string; + stck_prpr: string; + prdy_vrss: string; + prdy_vrss_sign: string; + prdy_ctrt: string; + acml_tr_pbmn: string; + acml_vol: string; + prdy_vrss_vol_rate: string; + stck_oprc: string; + stck_hgpr: string; + stck_lwpr: string; + stck_mxpr: string; + stck_llam: string; + stck_sdpr: string; + wghn_avrg_stck_prc: string; + hts_frgn_ehrt: string; + frgn_ntby_qty: string; + pgtr_ntby_qty: string; + pvt_scnd_dmrs_prc: string; + pvt_frst_dmrs_prc: string; + pvt_pont_val: string; + pvt_frst_dmsp_prc: string; + pvt_scnd_dmsp_prc: string; + dmrs_val: string; + dmsp_val: string; + cpfn: string; + rstc_wdth_prc: string; + stck_fcam: string; + stck_sspr: string; + aspr_unit: string; + hts_deal_qty_unit_val: string; + lstn_stcn: string; + hts_avls: string; + per: string; + pbr: string; + stac_month: string; + vol_tnrt: string; + eps: string; + bps: string; + d250_hgpr: string; + d250_hgpr_date: string; + d250_hgpr_vrss_prpr_rate: string; + d250_lwpr: string; + d250_lwpr_date: string; + d250_lwpr_vrss_prpr_rate: string; + stck_dryy_hgpr: string; + dryy_hgpr_vrss_prpr_rate: string; + dryy_hgpr_date: string; + stck_dryy_lwpr: string; + dryy_lwpr_vrss_prpr_rate: string; + dryy_lwpr_date: string; + w52_hgpr: string; + w52_hgpr_vrss_prpr_ctrt: string; + w52_hgpr_date: string; + w52_lwpr: string; + w52_lwpr_vrss_prpr_ctrt: string; + w52_lwpr_date: string; + whol_loan_rmnd_rate: string; + ssts_yn: string; + stck_shrn_iscd: string; + fcam_cnnm: string; + cpfn_cnnm: string; + frgn_hldn_qty: string; + vi_cls_code: string; + ovtm_vi_cls_code: string; + last_ssts_cntg_qty: string; + invt_caful_yn: string; + mrkt_warn_cls_code: string; + short_over_yn: string; + sltr_yn: string; +}; + +export const isOpenapiLiveData = (data: any): data is OpenapiLiveData => { + return ( + typeof data === 'object' && + data !== null && + typeof data.iscd_stat_cls_code === 'string' && + typeof data.marg_rate === 'string' && + typeof data.rprs_mrkt_kor_name === 'string' && + typeof data.bstp_kor_isnm === 'string' && + typeof data.temp_stop_yn === 'string' && + typeof data.oprc_rang_cont_yn === 'string' && + typeof data.clpr_rang_cont_yn === 'string' && + typeof data.crdt_able_yn === 'string' && + typeof data.stck_prpr === 'string' && + typeof data.prdy_ctrt === 'string' && + typeof data.acml_vol === 'string' && + typeof data.stck_oprc === 'string' && + typeof data.stck_hgpr === 'string' && + typeof data.stck_lwpr === 'string' && + typeof data.wghn_avrg_stck_prc === 'string' && + typeof data.stck_shrn_iscd === 'string' + ); +}; diff --git a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts index 6df0ca19..e9f0869d 100644 --- a/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts +++ b/packages/backend/src/scraper/openapi/type/openapiUtil.type.ts @@ -3,6 +3,7 @@ export type TR_ID = | 'FHKST03010200' | 'FHKST66430300' | 'HHKDB669107C0' + | 'FHKST01010100' | 'CTPF1002R'; export const TR_IDS: Record = { @@ -10,4 +11,5 @@ export const TR_IDS: Record = { MINUTE_DATA: 'FHKST03010200', FINANCIAL_DATA: 'FHKST66430300', PRODUCTION_DETAIL: 'CTPF1002R', + LIVE_DATA: 'FHKST01010100', }; diff --git a/packages/backend/src/stock/domain/stockLiveData.entity.ts b/packages/backend/src/stock/domain/stockLiveData.entity.ts index bf82f8d6..ea480d6b 100644 --- a/packages/backend/src/stock/domain/stockLiveData.entity.ts +++ b/packages/backend/src/stock/domain/stockLiveData.entity.ts @@ -31,9 +31,6 @@ export class StockLiveData { @Column({ type: 'decimal', precision: 15, scale: 2 }) open: number; - @Column({ type: 'decimal', precision: 15, scale: 2 }) - previousClose: number; - @UpdateDateColumn() @Column({ type: 'timestamp' }) updatedAt: Date; From ddccc0a37900fb27f723b817327ead04ca9bf305 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 22:48:42 +0900 Subject: [PATCH 109/112] =?UTF-8?q?=F0=9F=92=84=20style:=20=EC=95=88=20?= =?UTF-8?q?=EC=93=B0=EC=9D=B4=EB=8A=94=20=EA=B2=83=20=EB=B9=BC=EA=B8=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index 6ffccb8a..d2155492 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -5,7 +5,6 @@ import { openApiConfig } from '../config/openapi.config'; import { isOpenapiLiveData } from '../type/openapiLiveData.type'; import { TR_IDS } from '../type/openapiUtil.type'; import { getOpenApi } from '../util/openapiUtil.api'; -import { OpenapiTokenApi } from './openapiToken.api'; import { Stock } from '@/stock/domain/stock.entity'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @@ -15,7 +14,6 @@ export class OpenapiLiveData { '/uapi/domestic-stock/v1/quotations/inquire-ccnl'; constructor( private readonly datasource: DataSource, - private readonly config: OpenapiTokenApi, @Inject('winston') private readonly logger: Logger, ) {} From df5b6e64a28fede35ed12fb5e900508c29cb2ca0 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Mon, 25 Nov 2024 23:06:23 +0900 Subject: [PATCH 110/112] =?UTF-8?q?=F0=9F=92=84=20style:=20console.log=20?= =?UTF-8?q?=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts | 1 + packages/backend/src/scraper/openapi/liveData.service.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts index d2155492..7db12b2c 100644 --- a/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts @@ -42,6 +42,7 @@ export class OpenapiLiveData { } } + // 현재가 체결로는 데이터가 부족해 현재가 시세를 사용함. convertResponseToStockLiveData = ( data: OpenapiLiveData, stockId: string, diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 343540e2..68ed8c0f 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -87,7 +87,6 @@ export class LiveData { private initMessageCallback = (client: WebSocket) => async (data: RawData) => { try { - console.log(data); const message = this.parseMessage(data); if (message.header) { if (message.header.tr_id === 'PINGPONG') { From 79a34b644760f7c0b75f7adf049fdabfbb4e5186 Mon Sep 17 00:00:00 2001 From: sunghwki Date: Tue, 26 Nov 2024 12:59:51 +0900 Subject: [PATCH 111/112] =?UTF-8?q?=F0=9F=90=9B=20fix:=20pk=EA=B0=80=20?= =?UTF-8?q?=EC=95=84=EB=8B=8C=20=EA=B3=B3=EC=97=90=20unique=20=ED=82=A4=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/stock/domain/stockData.entity.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/stock/domain/stockData.entity.ts b/packages/backend/src/stock/domain/stockData.entity.ts index 7f6790d3..0a85083c 100644 --- a/packages/backend/src/stock/domain/stockData.entity.ts +++ b/packages/backend/src/stock/domain/stockData.entity.ts @@ -5,9 +5,11 @@ import { CreateDateColumn, JoinColumn, ManyToOne, + Unique, } from 'typeorm'; import { Stock } from './stock.entity'; +@Unique(['stock.id', 'startTime']) export class StockData { @PrimaryGeneratedColumn() id: number; From a7517569f295c16c230e54c412f0dc36703fc7ed Mon Sep 17 00:00:00 2001 From: sunghwki Date: Tue, 26 Nov 2024 13:00:24 +0900 Subject: [PATCH 112/112] =?UTF-8?q?=F0=9F=92=84=20style:=20=EB=B6=88?= =?UTF-8?q?=ED=95=84=EC=9A=94=ED=95=9C=20logger=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/scraper/openapi/api/openapiDetailData.api.ts | 10 ++++------ .../src/scraper/openapi/api/openapiPeriodData.api.ts | 2 +- .../backend/src/scraper/openapi/liveData.service.ts | 5 ----- .../openapi/websocket/websocketClient.websocket.ts | 2 -- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts index 982ebaf6..a544bc9b 100644 --- a/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiDetailData.api.ts @@ -30,9 +30,7 @@ export class OpenapiDetailData { private readonly openApiToken: OpenapiTokenApi, private readonly datasource: DataSource, @Inject('winston') private readonly logger: Logger, - ) { - //setTimeout(() => this.getDetailData(), 5000); - } + ) {} @Cron('0 8 * * 1-5') async getDetailData() { @@ -224,8 +222,8 @@ export class OpenapiDetailData { const output1 = await this.getFinancialRatio(stock, conf); const output2 = await this.getProductData(stock, conf); - this.logger.info(JSON.stringify(output1)); - this.logger.info(JSON.stringify(output2)); + console.log(output1); + console.log(output2); if (isFinancialRatioData(output1) && isProductDetail(output2)) { const stockDetail = await this.makeStockDetailObject( output1, @@ -236,7 +234,7 @@ export class OpenapiDetailData { const kospiStock = await this.makeKospiStockObject(output2, stock.id!); this.saveKospiData(kospiStock); - this.logger.info(`${stock.id!} is saved`); + this.logger.info(`${stock.id!} detail data is saved`); } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 5471cf5e..6fa3d5d0 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -157,7 +157,7 @@ export class OpenapiPeriodData { return await manager.findOne(entity, { where: { stock: { id: stock.stock.id }, - createdAt: stock.startTime, + startTime: stock.startTime, }, }); } diff --git a/packages/backend/src/scraper/openapi/liveData.service.ts b/packages/backend/src/scraper/openapi/liveData.service.ts index 68ed8c0f..f47ef1f6 100644 --- a/packages/backend/src/scraper/openapi/liveData.service.ts +++ b/packages/backend/src/scraper/openapi/liveData.service.ts @@ -30,7 +30,6 @@ export class LiveData { private async openapiSubscribe(stockId: string) { const config = (await this.openApiToken.configs())[0]; const result = await this.openapiLiveData.connectLiveData(stockId, config); - this.logger.info(JSON.stringify(result)); try { const stockLiveData = this.openapiLiveData.convertResponseToStockLiveData( result.output, @@ -90,13 +89,10 @@ export class LiveData { const message = this.parseMessage(data); if (message.header) { if (message.header.tr_id === 'PINGPONG') { - this.logger.info(`Received PING: ${data}`); client.pong(data); } return; } - this.logger.info(`Recived data : ${data}`); - this.logger.info(`Stock id : ${message[0]['STOCK_ID']}`); const liveData = this.openapiLiveData.convertLiveData(message); await this.openapiLiveData.saveLiveData(liveData); } catch (error) { @@ -142,7 +138,6 @@ export class LiveData { stockId: string, tr_type: TR_IDS, ): string { - this.logger.info(JSON.stringify(config)); const message = { header: { approval_key: config.STOCK_WEBSOCKET_KEY!, diff --git a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts index 06451a53..f9e4a5e3 100644 --- a/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts +++ b/packages/backend/src/scraper/openapi/websocket/websocketClient.websocket.ts @@ -12,12 +12,10 @@ export class WebsocketClient { constructor(@Inject('winston') private readonly logger: Logger) {} subscribe(message: string) { - this.logger.info(`Subscribe : ${message}`); this.sendMessage(message); } discribe(message: string) { - this.logger.info(`Discribe : ${message}`); this.sendMessage(message); }