feat: new deploy v2 protocol draft

package.json

@@ -44,7 +44,7 @@
     "@types/bcrypt": "^5.0.2",
     "@types/node": "^20.11.10",
     "@well-known-components/test-helpers": "^1.5.6",
-    "dcl-catalyst-client": "^21.7.0",
+    "dcl-catalyst-client": "https://sdk-team-cdn.decentraland.org/@dcl/catalyst-client/branch/deploy-v2/dcl-catalyst-client-21.7.1-10599703790.commit-1aa8621.tgz",
     "typescript": "^5.3.3"
   },
   "prettier": {

src/adapters/deployment-v2-manager.ts

@@ -0,0 +1,158 @@
+import { AppComponents, DeploymentResult } from '../types'
+import { AuthChain, Entity } from '@dcl/schemas'
+import { hashV1 } from '@dcl/hashing'
+import { InvalidRequestError } from '@dcl/platform-server-commons'
+
+export type IDeploymentV2Manager = {
+  initDeployment(
+    entityId: string,
+    entityRaw: Buffer,
+    authChain: AuthChain,
+    files: Record<string, number>
+  ): Promise<OngoingDeploymentMetadata>
+  addFileToDeployment(entityId: string, fileHash: string, file: Buffer): Promise<void>
+  completeDeployment(baseUrl: string, entityId: string): Promise<DeploymentResult>
+}
+
+export type StartDeploymentBody = { authChain: AuthChain; files: Record<string, number> }
+export type OngoingDeploymentMetadata = StartDeploymentBody & { availableFiles: string[]; missingFiles: string[] }
+
+export function createDeploymentV2Manager(
+  components: Pick<AppComponents, 'entityDeployer' | 'logs' | 'storage' | 'validator'>
+): IDeploymentV2Manager {
+  const { entityDeployer, logs, storage, validator } = components
+  const logger = logs.getLogger('deployment-v2-manager')
+  const ongoingDeploymentsRecord: Record<string, OngoingDeploymentMetadata> = {}
+  const tempFiles: Record<string, Buffer> = {}
+
+  async function initDeployment(
+    entityId: string,
+    entityRaw: Buffer,
+    authChain: AuthChain,
+    files: Record<string, number>
+  ): Promise<OngoingDeploymentMetadata> {
+    logger.info(`Init deployment for entity ${entityId}`)
+
+    // Check what files are already in storage and temporary storage and return the result
+    const results = Array.from((await storage.existMultiple(Object.keys(files))).entries())
+
+    // TODO we could check the local temp folder to avoid the client uploading the same file multiple times
+
+    const ongoingDeploymentMetadata = {
+      authChain,
+      files,
+      availableFiles: results.filter(([_, available]) => !!available).map(([cid, _]) => cid),
+      missingFiles: results.filter(([_, available]) => !available).map(([cid, _]) => cid)
+    }
+
+    // TODO we could store this in local file system of the ECS
+    ongoingDeploymentsRecord[entityId] = ongoingDeploymentMetadata
+
+    // TODO same
+    tempFiles[entityId] = entityRaw
+
+    return ongoingDeploymentMetadata
+  }
+
+  async function addFileToDeployment(entityId: string, fileHash: string, file: Buffer): Promise<void> {
+    const ongoingDeploymentsRecordElement = ongoingDeploymentsRecord[entityId]
+    if (!ongoingDeploymentsRecordElement) {
+      throw new InvalidRequestError(`Deployment for entity ${entityId} not found`)
+    }
+
+    // TODO we could check if the ongoing deployment is too old, and reject the file
+
+    const computedFileHash = await hashV1(file)
+    if (computedFileHash === entityId || fileHash === computedFileHash) {
+      if (!ongoingDeploymentsRecordElement.missingFiles.includes(fileHash)) {
+        throw new InvalidRequestError(`File with hash ${fileHash} not expected in deployment for entity ${entityId}`)
+      }
+
+      const expectedSize = ongoingDeploymentsRecordElement.files[fileHash]
+      if (expectedSize !== file.length) {
+        throw new InvalidRequestError(
+          `File with hash ${fileHash} has unexpected size in deployment for entity ${entityId}`
+        )
+      }
+    }
+
+    // TODO store in temp files folder
+    tempFiles[fileHash] = file
+    logger.info(`Received file ${fileHash} for entity ${entityId}`)
+  }
+
+  async function completeDeployment(baseUrl: string, entityId: string): Promise<DeploymentResult> {
+    try {
+      const ongoingDeploymentsRecordElement = ongoingDeploymentsRecord[entityId]
+      if (!ongoingDeploymentsRecordElement) {
+        throw new Error(`Deployment for entity ${entityId} not found`)
+      }
+
+      // TODO we could check if the ongoing deployment is too old, and reject the file (deployment TTL)
+
+      logger.info(`Completing deployment for entity ${entityId}`)
+
+      const authChain = ongoingDeploymentsRecordElement.authChain
+      const entityRaw = tempFiles[entityId].toString()
+      if (!entityRaw) {
+        throw new Error(`Entity file not found in deployment for entity ${entityId}.`)
+      }
+
+      const entityMetadataJson = JSON.parse(entityRaw.toString())
+
+      const entity: Entity = {
+        id: entityId, // this is not part of the published entity
+        timestamp: Date.now(), // this is not part of the published entity
+        ...entityMetadataJson
+      }
+
+      const uploadedFiles: Map<string, Uint8Array> = new Map()
+      for (const fileHash of Object.keys(ongoingDeploymentsRecordElement.files)) {
+        if (tempFiles[fileHash]) {
+          uploadedFiles.set(fileHash, tempFiles[fileHash])
+        }
+      }
+
+      const contentHashesInStorage = await storage.existMultiple(
+        Array.from(new Set(entity.content!.map(($) => $.hash)))
+      )
+
+      // run all validations about the deployment
+      const validationResult = await validator.validate({
+        entity,
+        files: uploadedFiles,
+        authChain,
+        contentHashesInStorage
+      })
+      if (!validationResult.ok()) {
+        throw new InvalidRequestError(`Deployment failed: ${validationResult.errors.join(', ')}`)
+      }
+
+      // Store the entity
+      return await entityDeployer.deployEntity(
+        baseUrl,
+        entity,
+        contentHashesInStorage,
+        uploadedFiles,
+        entityRaw,
+        authChain
+      )
+    } finally {
+      // Clean up temporary files
+      const ongoingDeploymentsRecordElement = ongoingDeploymentsRecord[entityId]
+
+      for (const fileHash in ongoingDeploymentsRecordElement.files) {
+        delete tempFiles[fileHash]
+      }
+
+      // Clean up ongoing deployments
+      delete ongoingDeploymentsRecord[entityId]
+    }
+  }
+
+  return {
+    initDeployment,
+    addFileToDeployment,
+    completeDeployment
+  }
+}

src/components.ts

@@ -22,7 +22,7 @@ import { createWorldsManagerComponent } from './adapters/worlds-manager'
 import { createCommsAdapterComponent } from './adapters/comms-adapter'
 import { createWorldsIndexerComponent } from './adapters/worlds-indexer'
 
-import { createValidator } from './logic/validations'
+import { createPreDeploymentValidator, createValidator } from './logic/validations'
 import { createEntityDeployer } from './adapters/entity-deployer'
 import { createMigrationExecutor } from './adapters/migration-executor'
 import { createNameDenyListChecker } from './adapters/name-deny-list-checker'
@@ -36,6 +36,7 @@ import { createSnsClient } from './adapters/sns-client'
 import { createAwsConfig } from './adapters/aws-config'
 import { S3 } from 'aws-sdk'
 import { createNotificationsClientComponent } from './adapters/notifications-service'
+import { createDeploymentV2Manager } from './adapters/deployment-v2-manager'
 
 // Initialize all the components of the app
 export async function initComponents(): Promise<AppComponents> {
@@ -134,6 +135,17 @@ export async function initComponents(): Promise<AppComponents> {
     worldsManager
   })
 
+  const preDeploymentValidator = createPreDeploymentValidator({
+    config,
+    nameDenyListChecker,
+    namePermissionChecker,
+    limitsManager,
+    storage,
+    worldsManager
+  })
+
+  const deploymentV2Manager = createDeploymentV2Manager({ entityDeployer, logs, storage, validator })
+
   const migrationExecutor = createMigrationExecutor({ logs, database: database, nameOwnership, storage, worldsManager })
 
   const notificationService = await createNotificationsClientComponent({ config, fetch, logs })
@@ -153,6 +165,7 @@ export async function initComponents(): Promise<AppComponents> {
     commsAdapter,
     config,
     database,
+    deploymentV2Manager,
     entityDeployer,
     ethereumProvider,
     fetch,
@@ -166,6 +179,7 @@ export async function initComponents(): Promise<AppComponents> {
     namePermissionChecker,
     notificationService,
     permissionsManager,
+    preDeploymentValidator,
     server,
     snsClient,
     status,

src/controllers/handlers/deploy-entity-handler.ts

@@ -1,9 +1,8 @@
-import { Entity } from '@dcl/schemas'
 import { IHttpServerComponent } from '@well-known-components/interfaces'
+import { InvalidRequestError } from '@dcl/platform-server-commons'
 import { FormDataContext } from '../../logic/multipart'
 import { HandlerContextWithPath } from '../../types'
-import { extractAuthChain } from '../../logic/extract-auth-chain'
-import { InvalidRequestError } from '@dcl/platform-server-commons'
+import { extractFromContext } from '../../logic/extract-deployment-info'
 
 export function requireString(val: string | null | undefined): string {
   if (typeof val !== 'string') throw new Error('A string was expected')
@@ -13,22 +12,7 @@ export function requireString(val: string | null | undefined): string {
 export async function deployEntity(
   ctx: FormDataContext & HandlerContextWithPath<'config' | 'entityDeployer' | 'storage' | 'validator', '/entities'>
 ): Promise<IHttpServerComponent.IResponse> {
-  const entityId = requireString(ctx.formData.fields.entityId.value)
-  const authChain = extractAuthChain(ctx)
-
-  const entityRaw = ctx.formData.files[entityId].value.toString()
-  const entityMetadataJson = JSON.parse(entityRaw)
-
-  const entity: Entity = {
-    id: entityId, // this is not part of the published entity
-    timestamp: Date.now(), // this is not part of the published entity
-    ...entityMetadataJson
-  }
-
-  const uploadedFiles: Map<string, Uint8Array> = new Map()
-  for (const filesKey in ctx.formData.files) {
-    uploadedFiles.set(filesKey, ctx.formData.files[filesKey].value)
-  }
+  const { authChain, entity, entityRaw, uploadedFiles } = extractFromContext(ctx)
 
   const contentHashesInStorage = await ctx.components.storage.existMultiple(
     Array.from(new Set(entity.content!.map(($) => $.hash)))
@@ -52,7 +36,7 @@ export async function deployEntity(
     entity,
     contentHashesInStorage,
     uploadedFiles,
-    entityRaw,
+    entityRaw.toString(),
     authChain
   )
 

src/controllers/handlers/deploy-v2-handlers.ts

@@ -0,0 +1,83 @@
+import { IHttpServerComponent } from '@well-known-components/interfaces'
+import { HandlerContextWithPath } from '../../types'
+import { InvalidRequestError } from '@dcl/platform-server-commons'
+import { FormDataContext } from '../../logic/multipart'
+import { extractFromContext } from '../../logic/extract-deployment-info'
+
+export function requireString(val: string | null | undefined): string {
+  if (typeof val !== 'string') throw new Error('A string was expected')
+  return val
+}
+
+export async function startDeployEntity(
+  ctx: FormDataContext &
+    HandlerContextWithPath<'config' | 'deploymentV2Manager' | 'storage' | 'preDeploymentValidator', '/v2/entities'>
+): Promise<IHttpServerComponent.IResponse> {
+  const { authChain, entity, entityRaw, uploadedFiles } = extractFromContext(ctx)
+
+  const contentHashesInStorage = await ctx.components.storage.existMultiple(
+    Array.from(new Set(entity.content!.map(($) => $.hash)))
+  )
+
+  const fileSizesManifest = JSON.parse(ctx.formData.fields.fileSizesManifest.value.toString())
+
+  // run all validations about the deployment
+  const validationResult = await ctx.components.preDeploymentValidator.validate({
+    entity,
+    files: uploadedFiles,
+    authChain,
+    contentHashesInStorage,
+    fileSizesManifest
+  })
+  if (!validationResult.ok()) {
+    throw new InvalidRequestError(`Deployment failed: ${validationResult.errors.join(', ')}`)
+  }
+
+  const ongoingDeploymentData = await ctx.components.deploymentV2Manager.initDeployment(
+    entity.id,
+    entityRaw,
+    authChain,
+    fileSizesManifest
+  )
+
+  return {
+    status: 200,
+    body: {
+      availableFiles: ongoingDeploymentData.availableFiles,
+      missingFiles: ongoingDeploymentData.missingFiles
+    }
+  }
+}
+
+export async function deployFile(
+  ctx: HandlerContextWithPath<'deploymentV2Manager', '/v2/entities/:entityId/files/:fileHash'>
+): Promise<IHttpServerComponent.IResponse> {
+  const entityId = ctx.params.entityId
+  const fileHash = ctx.params.fileHash
+  const buffer = await ctx.request.buffer()
+
+  await ctx.components.deploymentV2Manager.addFileToDeployment(entityId, fileHash, buffer)
+
+  return {
+    status: 204,
+    body: {}
+  }
+}
+
+export async function finishDeployEntity(
+  ctx: HandlerContextWithPath<
+    'config' | 'deploymentV2Manager' | 'entityDeployer' | 'storage' | 'validator',
+    '/v2/entities/:entityId'
+  >
+): Promise<IHttpServerComponent.IResponse> {
+  const baseUrl = (await ctx.components.config.getString('HTTP_BASE_URL')) || `https://${ctx.url.host}`
+
+  const message = await ctx.components.deploymentV2Manager.completeDeployment(baseUrl, ctx.params.entityId)
+  return {
+    status: 204,
+    body: {
+      creationTimestamp: Date.now(),
+      message
+    }
+  }
+}