Merge pull request #1064 from autonomys/add-download-file-ep

Add download file endpoint

indexers/api/package.json

@@ -39,6 +39,7 @@
     "@nestjs/typeorm": "^10.0.2",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
+    "mime-types": "^2.1.35",
     "passport": "^0.7.0",
     "passport-headerapikey": "^1.2.2",
     "pg": "^8.13.1",
@@ -53,6 +54,7 @@
     "@nestjs/testing": "^10.0.0",
     "@types/express": "^5.0.0",
     "@types/jest": "^29.5.2",
+    "@types/mime-types": "^2.1.4",
     "@types/node": "^20.3.1",
     "@types/supertest": "^6.0.0",
     "@typescript-eslint/eslint-plugin": "^8.0.0",

indexers/api/src/app.module.ts

@@ -6,6 +6,7 @@ import { AccountsController } from './controllers/accounts.controller';
 import { AppController } from './controllers/app.controller';
 import { BlocksController } from './controllers/blocks.controller';
 import { ExtrinsicsController } from './controllers/extrinsics.controller';
+import { FilesController } from './controllers/files.controller';
 import {
   Accounts,
   ApiDailyUsage,
@@ -14,14 +15,18 @@ import {
   ApiKeysMonthlyUsage,
   ApiMonthlyUsage,
   Blocks,
+  Chunks,
   ConsensusMetadata,
   Extrinsics,
+  FileCids,
+  Files,
   FilesMetadata,
   LeaderboardMetadata,
   Profile,
   StakingMetadata,
 } from './entities';
 import { ApiUsageService } from './services/api-usage.service';
+import { FileRetrieverService } from './services/file-retriever.sevice';
 
 @Module({
   imports: [
@@ -42,6 +47,9 @@ import { ApiUsageService } from './services/api-usage.service';
         ApiKeysDailyUsage,
         ApiKeysMonthlyUsage,
         Accounts,
+        Files,
+        Chunks,
+        FileCids,
         ConsensusMetadata,
         LeaderboardMetadata,
         FilesMetadata,
@@ -59,6 +67,9 @@ import { ApiUsageService } from './services/api-usage.service';
       ApiKeysDailyUsage,
       ApiKeysMonthlyUsage,
       Accounts,
+      Files,
+      Chunks,
+      FileCids,
       ConsensusMetadata,
       LeaderboardMetadata,
       FilesMetadata,
@@ -71,7 +82,8 @@ import { ApiUsageService } from './services/api-usage.service';
     BlocksController,
     ExtrinsicsController,
     AccountsController,
+    FilesController,
   ],
-  providers: [ApiKeyStrategy, ApiUsageService],
+  providers: [ApiKeyStrategy, ApiUsageService, FileRetrieverService],
 })
 export class AppModule {}

indexers/api/src/controllers/files.controller.ts

@@ -0,0 +1,72 @@
+import { Controller, Get, NotFoundException, Param, Res } from '@nestjs/common';
+import { ApiOperation, ApiParam, ApiResponse, ApiTags } from '@nestjs/swagger';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Response } from 'express';
+import { Repository } from 'typeorm';
+import { Chunks } from '../entities/files/chunks.entity';
+import { Files } from '../entities/files/files.entity';
+import { FileRetrieverService } from '../services/file-retriever.sevice';
+
+@ApiTags('Files')
+@Controller('files')
+export class FilesController {
+  constructor(
+    private fileRetrieverService: FileRetrieverService,
+    @InjectRepository(Files)
+    private filesRepository: Repository<Files>,
+    @InjectRepository(Chunks)
+    private chunksRepository: Repository<Chunks>,
+  ) {}
+
+  @Get(':cid')
+  @ApiOperation({
+    operationId: 'getFile',
+    summary: 'Download file by CID',
+  })
+  @ApiParam({ name: 'cid', description: 'CID of the file' })
+  @ApiResponse({
+    status: 200,
+    description: 'Returns the file content as a byte stream',
+    headers: {
+      'Content-Type': {
+        description: 'The MIME type of the file',
+        example: 'application/octet-stream',
+      },
+    },
+  })
+  async getFile(
+    @Param('cid') cid: string,
+    @Res() res: Response,
+  ): Promise<void> {
+    const file = await this.filesRepository.findOne({
+      where: {
+        id: cid,
+      },
+    });
+    if (!file) {
+      throw new NotFoundException(`File with CID ${cid} not found`);
+    }
+
+    const chunk = await this.chunksRepository.findOne({
+      where: {
+        id: cid,
+      },
+    });
+
+    const contentType = file.contentType() || 'application/octet-stream';
+    res.setHeader('Content-Type', contentType);
+    res.setHeader('Content-Length', file.size);
+    res.setHeader('Content-Disposition', `filename="${file.name}"`);
+
+    const isCompressedAndPlainText =
+      chunk?.upload_options?.encryption?.algorithm === undefined &&
+      chunk?.upload_options?.compression?.algorithm !== undefined;
+    if (isCompressedAndPlainText) {
+      res.setHeader('Content-Encoding', 'deflate');
+    }
+
+    const fileBuffer = await this.fileRetrieverService.getBuffer(cid);
+
+    res.send(fileBuffer);
+  }
+}

indexers/api/src/entities/files/chunks.entity.ts

@@ -0,0 +1,46 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Column, Entity } from 'typeorm';
+import { BaseEntity } from '../consensus/base.entity';
+import { UploadOptions } from './uploadOptions.valueObject';
+
+@Entity('chunks', { schema: 'files' })
+export class Chunks extends BaseEntity {
+  @ApiProperty()
+  @Column('text')
+  id: string;
+
+  @ApiProperty()
+  @Column('enum', {
+    enum: [
+      'FileChunk',
+      'File',
+      'Folder',
+      'FileInlink',
+      'FolderInlink',
+      'Metadata',
+      'MetadataInlink',
+      'MetadataChunk',
+    ],
+  })
+  type: string;
+
+  @ApiProperty()
+  @Column('numeric')
+  link_depth: number;
+
+  @ApiProperty()
+  @Column('numeric')
+  size: number;
+
+  @ApiProperty()
+  @Column('varchar', { length: 255 })
+  name: string;
+
+  @ApiProperty()
+  @Column('text')
+  data: string;
+
+  @ApiProperty()
+  @Column('jsonb')
+  upload_options: UploadOptions;
+}

indexers/api/src/entities/files/file-cids.entity.ts

@@ -0,0 +1,18 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Column, Entity } from 'typeorm';
+import { BaseEntity } from '../consensus/base.entity';
+
+@Entity('file_cids', { schema: 'files' })
+export class FileCids extends BaseEntity {
+  @ApiProperty()
+  @Column('varchar', { length: 161 })
+  id: string;
+
+  @ApiProperty()
+  @Column('varchar', { length: 161 })
+  parent_cid: string;
+
+  @ApiProperty()
+  @Column('varchar', { length: 161 })
+  child_cid: string;
+}

indexers/api/src/entities/files/files.entity.ts

@@ -0,0 +1,26 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { lookup } from 'mime-types';
+import { AfterInsert, AfterLoad, AfterUpdate, Column, Entity } from 'typeorm';
+import { BaseEntity } from '../consensus/base.entity';
+
+@Entity('files', { schema: 'files' })
+export class Files extends BaseEntity {
+  @ApiProperty()
+  @Column('varchar', { length: 80 })
+  id: string;
+
+  @ApiProperty()
+  @Column('numeric')
+  size: number;
+
+  @ApiProperty()
+  @Column('varchar', { length: 255, nullable: true })
+  name: string;
+
+  @AfterLoad()
+  @AfterInsert()
+  @AfterUpdate()
+  contentType(): string | undefined {
+    return lookup(this.name) || undefined;
+  }
+}

indexers/api/src/entities/files/uploadOptions.valueObject.ts

@@ -0,0 +1,23 @@
+export type UploadOptions = {
+  compression?: CompressionOptions;
+  encryption?: EncryptionOptions;
+};
+
+export type CompressionOptions = {
+  algorithm: CompressionAlgorithm;
+  level?: number;
+  chunkSize?: number;
+};
+
+export type EncryptionOptions = {
+  algorithm: EncryptionAlgorithm;
+  chunkSize?: number;
+};
+
+export enum EncryptionAlgorithm {
+  AES_256_GCM = 'AES_256_GCM',
+}
+
+export enum CompressionAlgorithm {
+  ZLIB = 'ZLIB',
+}

indexers/api/src/entities/index.ts

@@ -26,6 +26,9 @@ export * from './consensus/transfers.entity';
 export * from './leaderboard/metadata.entity';
 
 // Files Entities
+export * from './files/chunks.entity';
+export * from './files/file-cids.entity';
+export * from './files/files.entity';
 export * from './files/metadata.entity';
 
 // Staking Entities

indexers/api/src/services/file-retriever.sevice.ts

@@ -0,0 +1,37 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { In, Repository } from 'typeorm';
+import { Chunks } from '../entities/files/chunks.entity';
+import { FileCids } from '../entities/files/file-cids.entity';
+
+@Injectable()
+export class FileRetrieverService {
+  constructor(
+    @InjectRepository(FileCids)
+    private fileCidsRepository: Repository<FileCids>,
+    @InjectRepository(Chunks)
+    private chunksRepository: Repository<Chunks>,
+  ) {}
+
+  async getBuffer(fileId: string): Promise<Buffer> {
+    const children = await this.fileCidsRepository.find({
+      where: {
+        parent_cid: fileId,
+      },
+    });
+
+    const chunks = await this.chunksRepository.find({
+      where: {
+        id: In(children.map((child) => child.child_cid)),
+      },
+    });
+
+    return Buffer.concat(
+      chunks.map((chunk) =>
+        Buffer.from(
+          Object.values(JSON.parse(chunk.data) as Record<string, number>),
+        ),
+      ),
+    );
+  }
+}

indexers/api/yarn.lock

@@ -1001,6 +1001,11 @@
   resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547"
   integrity sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==
 
+"@types/mime-types@^2.1.4":
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/@types/mime-types/-/mime-types-2.1.4.tgz#93a1933e24fed4fb9e4adc5963a63efcbb3317a2"
+  integrity sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==
+
 "@types/mime@^1":
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690"
@@ -3683,7 +3688,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.24, mime-types@~2.1.34:
+mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.35, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==