feat: prevent re-converting scenes that were already converted during the last 3 days

consumer-server/src/adapters/storage.ts

@@ -12,7 +12,7 @@ export async function createCloudStorageAdapter({ config }: Pick<AppComponents,
 
   const s3 = new S3Client({ region, endpoint: bucketEndpoint })
 
-  async function getFiles(prefix: string): Promise<string[]> {
+  async function getFiles(prefix: string): Promise<{ key: string, lastModified: Date | undefined}[]> {
     const list = await s3.send(
       new ListObjectsV2Command({
         Bucket: bucket,
@@ -22,7 +22,7 @@ export async function createCloudStorageAdapter({ config }: Pick<AppComponents,
 
     return (
       list.Contents?.filter((file) => !file.Key?.endsWith('output.txt')).map(
-        (file) => `${bucketEndpoint}/${bucket}/${file.Key}`
+        (file) => ({ key: `${bucketEndpoint}/${bucket}/${file.Key}`, lastModified: file.LastModified })
       ) || []
     )
   }

consumer-server/src/logic/message-processor.ts

@@ -65,6 +65,29 @@ export async function createMessageProcesorComponent({
         return
       }
 
+      const alreadyUploadedFiles = await storage.getFiles(`${base}/LOD/Sources/${message.entity.entityTimestamp.toString()}`)
+
+      if (!!alreadyUploadedFiles.length) {
+          const lastUploadDate = alreadyUploadedFiles.reduce((acc, file) => {
+            if (!file.lastModified) return acc
+            return file.lastModified > acc ? file.lastModified : acc
+          }, new Date(0))
+
+          const currentDate = new Date()
+          const diff = currentDate.getTime() - lastUploadDate.getTime()
+          const diffDays = diff / (1000 * 3600 * 24)
+          if (diffDays < 3) {
+            logger.debug('Skipping process since it was already processed within the last 3 days', {
+              entityId,
+              base,
+              lastUploadDate: lastUploadDate.toISOString(),
+              currentDate: currentDate.toISOString()
+            })
+            await queue.deleteMessage(receiptMessageHandle)
+            return
+          }
+      }
+
       logger.info('Processing scene deployment', {
         entityId,
         base,

consumer-server/src/types.ts

@@ -102,7 +102,7 @@ export type MessageProcessorComponent = {
 
 export type StorageComponent = {
   storeFiles(filePaths: string[], prefix: string): Promise<string[]>
-  getFiles(prefix: string): Promise<string[]>
+  getFiles(prefix: string): Promise<{ key: string, lastModified: Date | undefined}[]>
   deleteFailureDirectory(pointer: string): Promise<void>
 }