diff --git a/indexers/api/package.json b/indexers/api/package.json index 35868e70..a28ff747 100644 --- a/indexers/api/package.json +++ b/indexers/api/package.json @@ -39,6 +39,7 @@ "@nestjs/typeorm": "^10.0.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "mime-types": "^2.1.35", "passport": "^0.7.0", "passport-headerapikey": "^1.2.2", "pg": "^8.13.1", @@ -53,6 +54,7 @@ "@nestjs/testing": "^10.0.0", "@types/express": "^5.0.0", "@types/jest": "^29.5.2", + "@types/mime-types": "^2.1.4", "@types/node": "^20.3.1", "@types/supertest": "^6.0.0", "@typescript-eslint/eslint-plugin": "^8.0.0", diff --git a/indexers/api/src/app.module.ts b/indexers/api/src/app.module.ts index 6eb5c994..8fe32541 100644 --- a/indexers/api/src/app.module.ts +++ b/indexers/api/src/app.module.ts @@ -6,6 +6,7 @@ import { AccountsController } from './controllers/accounts.controller'; import { AppController } from './controllers/app.controller'; import { BlocksController } from './controllers/blocks.controller'; import { ExtrinsicsController } from './controllers/extrinsics.controller'; +import { FilesController } from './controllers/files.controller'; import { Accounts, ApiDailyUsage, @@ -14,14 +15,18 @@ import { ApiKeysMonthlyUsage, ApiMonthlyUsage, Blocks, + Chunks, ConsensusMetadata, Extrinsics, + FileCids, + Files, FilesMetadata, LeaderboardMetadata, Profile, StakingMetadata, } from './entities'; import { ApiUsageService } from './services/api-usage.service'; +import { FileRetrieverService } from './services/file-retriever.sevice'; @Module({ imports: [ @@ -42,6 +47,9 @@ import { ApiUsageService } from './services/api-usage.service'; ApiKeysDailyUsage, ApiKeysMonthlyUsage, Accounts, + Files, + Chunks, + FileCids, ConsensusMetadata, LeaderboardMetadata, FilesMetadata, @@ -59,6 +67,9 @@ import { ApiUsageService } from './services/api-usage.service'; ApiKeysDailyUsage, ApiKeysMonthlyUsage, Accounts, + Files, + Chunks, + FileCids, ConsensusMetadata, LeaderboardMetadata, FilesMetadata, @@ -71,7 +82,8 @@ import { ApiUsageService } from './services/api-usage.service'; BlocksController, ExtrinsicsController, AccountsController, + FilesController, ], - providers: [ApiKeyStrategy, ApiUsageService], + providers: [ApiKeyStrategy, ApiUsageService, FileRetrieverService], }) export class AppModule {} diff --git a/indexers/api/src/controllers/files.controller.ts b/indexers/api/src/controllers/files.controller.ts new file mode 100644 index 00000000..d061ae5e --- /dev/null +++ b/indexers/api/src/controllers/files.controller.ts @@ -0,0 +1,72 @@ +import { Controller, Get, NotFoundException, Param, Res } from '@nestjs/common'; +import { ApiOperation, ApiParam, ApiResponse, ApiTags } from '@nestjs/swagger'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Response } from 'express'; +import { Repository } from 'typeorm'; +import { Chunks } from '../entities/files/chunks.entity'; +import { Files } from '../entities/files/files.entity'; +import { FileRetrieverService } from '../services/file-retriever.sevice'; + +@ApiTags('Files') +@Controller('files') +export class FilesController { + constructor( + private fileRetrieverService: FileRetrieverService, + @InjectRepository(Files) + private filesRepository: Repository, + @InjectRepository(Chunks) + private chunksRepository: Repository, + ) {} + + @Get(':cid') + @ApiOperation({ + operationId: 'getFile', + summary: 'Download file by CID', + }) + @ApiParam({ name: 'cid', description: 'CID of the file' }) + @ApiResponse({ + status: 200, + description: 'Returns the file content as a byte stream', + headers: { + 'Content-Type': { + description: 'The MIME type of the file', + example: 'application/octet-stream', + }, + }, + }) + async getFile( + @Param('cid') cid: string, + @Res() res: Response, + ): Promise { + const file = await this.filesRepository.findOne({ + where: { + id: cid, + }, + }); + if (!file) { + throw new NotFoundException(`File with CID ${cid} not found`); + } + + const chunk = await this.chunksRepository.findOne({ + where: { + id: cid, + }, + }); + + const contentType = file.contentType() || 'application/octet-stream'; + res.setHeader('Content-Type', contentType); + res.setHeader('Content-Length', file.size); + res.setHeader('Content-Disposition', `filename="${file.name}"`); + + const isCompressedAndPlainText = + chunk?.upload_options?.encryption?.algorithm === undefined && + chunk?.upload_options?.compression?.algorithm !== undefined; + if (isCompressedAndPlainText) { + res.setHeader('Content-Encoding', 'deflate'); + } + + const fileBuffer = await this.fileRetrieverService.getBuffer(cid); + + res.send(fileBuffer); + } +} diff --git a/indexers/api/src/entities/files/chunks.entity.ts b/indexers/api/src/entities/files/chunks.entity.ts new file mode 100644 index 00000000..49a11ebb --- /dev/null +++ b/indexers/api/src/entities/files/chunks.entity.ts @@ -0,0 +1,46 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Column, Entity } from 'typeorm'; +import { BaseEntity } from '../consensus/base.entity'; +import { UploadOptions } from './uploadOptions.valueObject'; + +@Entity('chunks', { schema: 'files' }) +export class Chunks extends BaseEntity { + @ApiProperty() + @Column('text') + id: string; + + @ApiProperty() + @Column('enum', { + enum: [ + 'FileChunk', + 'File', + 'Folder', + 'FileInlink', + 'FolderInlink', + 'Metadata', + 'MetadataInlink', + 'MetadataChunk', + ], + }) + type: string; + + @ApiProperty() + @Column('numeric') + link_depth: number; + + @ApiProperty() + @Column('numeric') + size: number; + + @ApiProperty() + @Column('varchar', { length: 255 }) + name: string; + + @ApiProperty() + @Column('text') + data: string; + + @ApiProperty() + @Column('jsonb') + upload_options: UploadOptions; +} diff --git a/indexers/api/src/entities/files/file-cids.entity.ts b/indexers/api/src/entities/files/file-cids.entity.ts new file mode 100644 index 00000000..d21b74b4 --- /dev/null +++ b/indexers/api/src/entities/files/file-cids.entity.ts @@ -0,0 +1,18 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Column, Entity } from 'typeorm'; +import { BaseEntity } from '../consensus/base.entity'; + +@Entity('file_cids', { schema: 'files' }) +export class FileCids extends BaseEntity { + @ApiProperty() + @Column('varchar', { length: 161 }) + id: string; + + @ApiProperty() + @Column('varchar', { length: 161 }) + parent_cid: string; + + @ApiProperty() + @Column('varchar', { length: 161 }) + child_cid: string; +} diff --git a/indexers/api/src/entities/files/files.entity.ts b/indexers/api/src/entities/files/files.entity.ts new file mode 100644 index 00000000..1cc6ebb3 --- /dev/null +++ b/indexers/api/src/entities/files/files.entity.ts @@ -0,0 +1,26 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { lookup } from 'mime-types'; +import { AfterInsert, AfterLoad, AfterUpdate, Column, Entity } from 'typeorm'; +import { BaseEntity } from '../consensus/base.entity'; + +@Entity('files', { schema: 'files' }) +export class Files extends BaseEntity { + @ApiProperty() + @Column('varchar', { length: 80 }) + id: string; + + @ApiProperty() + @Column('numeric') + size: number; + + @ApiProperty() + @Column('varchar', { length: 255, nullable: true }) + name: string; + + @AfterLoad() + @AfterInsert() + @AfterUpdate() + contentType(): string | undefined { + return lookup(this.name) || undefined; + } +} diff --git a/indexers/api/src/entities/files/uploadOptions.valueObject.ts b/indexers/api/src/entities/files/uploadOptions.valueObject.ts new file mode 100644 index 00000000..9d716ab5 --- /dev/null +++ b/indexers/api/src/entities/files/uploadOptions.valueObject.ts @@ -0,0 +1,23 @@ +export type UploadOptions = { + compression?: CompressionOptions; + encryption?: EncryptionOptions; +}; + +export type CompressionOptions = { + algorithm: CompressionAlgorithm; + level?: number; + chunkSize?: number; +}; + +export type EncryptionOptions = { + algorithm: EncryptionAlgorithm; + chunkSize?: number; +}; + +export enum EncryptionAlgorithm { + AES_256_GCM = 'AES_256_GCM', +} + +export enum CompressionAlgorithm { + ZLIB = 'ZLIB', +} diff --git a/indexers/api/src/entities/index.ts b/indexers/api/src/entities/index.ts index 0e0fafef..ed8a90d5 100644 --- a/indexers/api/src/entities/index.ts +++ b/indexers/api/src/entities/index.ts @@ -26,6 +26,9 @@ export * from './consensus/transfers.entity'; export * from './leaderboard/metadata.entity'; // Files Entities +export * from './files/chunks.entity'; +export * from './files/file-cids.entity'; +export * from './files/files.entity'; export * from './files/metadata.entity'; // Staking Entities diff --git a/indexers/api/src/services/file-retriever.sevice.ts b/indexers/api/src/services/file-retriever.sevice.ts new file mode 100644 index 00000000..9e0ea51b --- /dev/null +++ b/indexers/api/src/services/file-retriever.sevice.ts @@ -0,0 +1,37 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { In, Repository } from 'typeorm'; +import { Chunks } from '../entities/files/chunks.entity'; +import { FileCids } from '../entities/files/file-cids.entity'; + +@Injectable() +export class FileRetrieverService { + constructor( + @InjectRepository(FileCids) + private fileCidsRepository: Repository, + @InjectRepository(Chunks) + private chunksRepository: Repository, + ) {} + + async getBuffer(fileId: string): Promise { + const children = await this.fileCidsRepository.find({ + where: { + parent_cid: fileId, + }, + }); + + const chunks = await this.chunksRepository.find({ + where: { + id: In(children.map((child) => child.child_cid)), + }, + }); + + return Buffer.concat( + chunks.map((chunk) => + Buffer.from( + Object.values(JSON.parse(chunk.data) as Record), + ), + ), + ); + } +} diff --git a/indexers/api/yarn.lock b/indexers/api/yarn.lock index fd86d4ce..decd6dac 100644 --- a/indexers/api/yarn.lock +++ b/indexers/api/yarn.lock @@ -1001,6 +1001,11 @@ resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" integrity sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ== +"@types/mime-types@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@types/mime-types/-/mime-types-2.1.4.tgz#93a1933e24fed4fb9e4adc5963a63efcbb3317a2" + integrity sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w== + "@types/mime@^1": version "1.3.5" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690" @@ -3683,7 +3688,7 @@ mime-db@1.52.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.35, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==