Merge branch 'master' into develop

# Conflicts:
#	Dockerfile

.github/workflows/build-rc-image.yaml

@@ -39,4 +39,5 @@ jobs:
       build-platforms: ${{ needs.variables-setup.outputs.platforms }}
       version: ${{ needs.variables-setup.outputs.version }}
       date: ${{ needs.variables-setup.outputs.date }}
+      scan-image: true
     secrets: inherit

Dockerfile

@@ -53,16 +53,9 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" PYTHONPATH=/backend \
     UWSGI_LAZY_APPS=1 UWSGI_WSGI_ENV_BEHAVIOR=holy PYTHONDONTWRITEBYTECODE=1
 
 RUN dnf -y upgrade && dnf -y install python3.11 python3.11-pip ca-certificates pcre-devel \
+    && dnf -y autoremove \
     && dnf clean all \
-    && groupadd uwsgi && useradd -g uwsgi uwsgi \
-    && chown -R uwsgi:uwsgi ${VIRTUAL_ENV} \
-    && chown -R uwsgi:uwsgi /usr/share/nltk_data \
-    && chown -R uwsgi:uwsgi /backend
-
-USER uwsgi
-EXPOSE 5001
-
-RUN mkdir -p -m 0644 /backend/storage \
+    && mkdir -p -m 0744 /backend/storage \
     && source "${VIRTUAL_ENV}/bin/activate" \
     && pip install --upgrade pip \
     && pip install --upgrade setuptools

app/commons/model/launch_objects.py

@@ -55,7 +55,7 @@ class ApplicationConfig(BaseModel):
     minioUseTls: bool = False
     appVersion: str = ''
     binaryStoreType: str = 'filesystem'
-    minioBucketPrefix: str = 'prj-'
+    bucketPrefix: str = 'prj-'
     minioRegion: str | None = None
     instanceTaskType: str = ''
     filesystemDefaultPath: str = 'storage'

app/commons/object_saving/__init__.py

@@ -24,8 +24,7 @@ def create(app_config: ApplicationConfig, project_id: str | int | None = None, p
 
 def create_filesystem(base_path: str, project_id: str | int | None = None, path: str | None = None) -> ObjectSaver:
     return ObjectSaver(
-        app_config=ApplicationConfig(binaryStoreType='filesystem', filesystemDefaultPath=base_path,
-                                     minioBucketPrefix=''),
+        app_config=ApplicationConfig(binaryStoreType='filesystem', filesystemDefaultPath=base_path, bucketPrefix=''),
         project_id=project_id,
         path=path
     )

app/commons/object_saving/filesystem_saver.py

@@ -20,67 +20,66 @@
 
 from app.commons import logging
 from app.commons.model.launch_objects import ApplicationConfig
-from app.commons.object_saving.storage import Storage
+from app.commons.object_saving.storage import Storage, unify_path_separator
 from app.utils import utils
 
 logger = logging.getLogger('analyzerApp.filesystemSaver')
 
 
-def unify_path_separator(path: str) -> str:
-    return path.replace('\\', '/')
-
-
 class FilesystemSaver(Storage):
     _base_path: str
 
     def __init__(self, app_config: ApplicationConfig) -> None:
-        self._base_path = app_config.filesystemDefaultPath
+        super().__init__(app_config)
+        self._base_path = app_config.filesystemDefaultPath or os.getcwd()
 
-    def remove_project_objects(self, path: str, object_names: list[str]) -> None:
+    def get_path(self, object_name: str, bucket) -> str:
+        return unify_path_separator(str(os.path.join(self._base_path, self._get_project_name(bucket), object_name)))
+
+    def remove_project_objects(self, bucket: str, object_names: list[str]) -> None:
         for filename in object_names:
-            object_name_full = unify_path_separator(os.path.join(self._base_path, path, filename))
+            object_name_full = self.get_path(filename, bucket)
             if os.path.exists(object_name_full):
                 os.remove(object_name_full)
 
-    def put_project_object(self, data: Any, path: str, object_name: str, using_json: bool = False) -> None:
-        folder_to_save = unify_path_separator(os.path.join(self._base_path, path, os.path.dirname(object_name)))
-        filename = unify_path_separator(os.path.join(self._base_path, path, object_name))
+    def put_project_object(self, data: Any, bucket: str, object_name: str, using_json: bool = False) -> None:
+        path = self.get_path(object_name, bucket)
+        folder_to_save = unify_path_separator(os.path.dirname(path))
         if folder_to_save:
             os.makedirs(folder_to_save, exist_ok=True)
-        with open(filename, 'wb') as f:
+        with open(path, 'wb') as f:
             if using_json:
                 f.write(json.dumps(data).encode('utf-8'))
             else:
                 # noinspection PyTypeChecker
                 pickle.dump(data, f)
-        logger.debug("Saved into folder '%s' with name '%s': %s", path, object_name, data)
+        logger.debug('Saved into folder "%s" with name "%s": %s', bucket, object_name, data)
 
-    def get_project_object(self, path: str, object_name: str, using_json: bool = False) -> object | None:
-        filename = unify_path_separator(os.path.join(self._base_path, path, object_name))
+    def get_project_object(self, bucket: str, object_name: str, using_json: bool = False) -> object | None:
+        filename = self.get_path(object_name, bucket)
         if not utils.validate_file(filename):
             raise ValueError(f'Unable to get file: {filename}')
         with open(filename, 'rb') as f:
             return json.loads(f.read()) if using_json else pickle.load(f)
 
-    def does_object_exists(self, path: str, object_name: str) -> bool:
-        return os.path.exists(unify_path_separator(os.path.join(self._base_path, path, object_name)))
+    def does_object_exists(self, bucket: str, object_name: str) -> bool:
+        return os.path.exists(self.get_path(object_name, bucket))
 
-    def get_folder_objects(self, path: str, folder: str) -> list[str]:
-        root_path = self._base_path
-        if not root_path and not path:
-            root_path = os.getcwd()
+    def get_folder_objects(self, bucket: str, folder: str) -> list[str]:
+        path = self.get_path(folder, bucket)
         if unify_path_separator(folder).endswith('/'):
-            folder_to_check = unify_path_separator(os.path.join(root_path, path, folder))
-            if os.path.exists(folder_to_check):
-                return [os.path.join(folder, file_name) for file_name in os.listdir(folder_to_check)]
+            # The "folder" is a folder, list it
+            if os.path.exists(path):
+                return [os.path.join(folder, file_name) for file_name in os.listdir(path)]
         else:
-            folder_to_check = unify_path_separator(os.path.join(root_path, path))
+            # The "folder" is a filename pattern, list base folder and filter files
+            folder_to_check = unify_path_separator(os.path.dirname(path))
             if os.path.exists(folder_to_check):
                 return [file_name for file_name in os.listdir(folder_to_check) if file_name.startswith(folder)]
         return []
 
-    def remove_folder_objects(self, path: str, folder: str) -> bool:
-        folder_name = unify_path_separator(os.path.join(self._base_path, path, folder))
+    def remove_folder_objects(self, bucket: str, folder: str) -> bool:
+        folder_name = self.get_path(folder, bucket)
         if os.path.exists(folder_name):
             shutil.rmtree(folder_name, ignore_errors=True)
             return True

app/commons/object_saving/minio_client.py

@@ -23,20 +23,19 @@
 
 from app.commons import logging
 from app.commons.model.launch_objects import ApplicationConfig
-from app.commons.object_saving.storage import Storage
+from app.commons.object_saving.storage import Storage, unify_path_separator
 
 logger = logging.getLogger("analyzerApp.minioClient")
 
 
 class MinioClient(Storage):
     region: str
-    bucket_prefix: str
     minio_client: Minio
 
     def __init__(self, app_config: ApplicationConfig) -> None:
+        super().__init__(app_config)
         minio_host = app_config.minioHost
         self.region = app_config.minioRegion
-        self.bucket_prefix = app_config.minioBucketPrefix
         self.minio_client = Minio(
             minio_host,
             access_key=app_config.minioAccessKey,
@@ -46,11 +45,6 @@ def __init__(self, app_config: ApplicationConfig) -> None:
         )
         logger.info(f'Minio initialized {minio_host}')
 
-    def _get_project_name(self, project_id: str | None) -> str:
-        if not project_id:
-            return ''
-        return self.bucket_prefix + project_id
-
     def get_bucket(self, bucket_id: str | None) -> str:
         path = self._get_project_name(bucket_id)
         if not path:
@@ -67,7 +61,7 @@ def get_path(self, object_name: str, bucket_name: str, bucket_id: str | None) ->
             return object_name
 
         path_octets = os.path.split(path)[1:]
-        return str(os.path.join(path_octets[0], *path_octets[1:], object_name))
+        return unify_path_separator(str(os.path.join(path_octets[0], *path_octets[1:], object_name)))
 
     def remove_project_objects(self, bucket: str, object_names: list[str]) -> None:
         bucket_name = self.get_bucket(bucket)

app/commons/object_saving/storage.py

@@ -13,12 +13,26 @@
 #  limitations under the License.
 
 """Common interface class for Storage types."""
-
 from abc import ABCMeta, abstractmethod
 from typing import Any
 
+from app.commons.model.launch_objects import ApplicationConfig
+
+
+def unify_path_separator(path: str) -> str:
+    return path.replace('\\', '/')
+
 
 class Storage(metaclass=ABCMeta):
+    _bucket_prefix: str
+
+    def __init__(self, app_config: ApplicationConfig) -> None:
+        self._bucket_prefix = app_config.bucketPrefix
+
+    def _get_project_name(self, project_id: str | None) -> str:
+        if not project_id:
+            return ''
+        return self._bucket_prefix + project_id
 
     @abstractmethod
     def remove_project_objects(self, path: str, object_names: list[str]) -> None:

app/main.py

@@ -68,12 +68,9 @@
     minioSecretKey=os.getenv("MINIO_SECRET_KEY", "minio123"),
     minioUseTls=json.loads(os.getenv("MINIO_USE_TLS", "false").lower()),
     appVersion="",
-    binaryStoreType=os.getenv("ANALYZER_BINSTORE_TYPE",
-                              os.getenv("ANALYZER_BINARYSTORE_TYPE", "filesystem")),
-    minioBucketPrefix=os.getenv("ANALYZER_BINSTORE_BUCKETPREFIX",
-                                os.getenv("ANALYZER_BINARYSTORE_BUCKETPREFIX", "prj-")),
-    minioRegion=os.getenv("ANALYZER_BINSTORE_MINIO_REGION",
-                          os.getenv("ANALYZER_BINARYSTORE_MINIO_REGION", None)),
+    binaryStoreType=os.getenv("ANALYZER_BINSTORE_TYPE", os.getenv("ANALYZER_BINARYSTORE_TYPE", "filesystem")),
+    bucketPrefix=os.getenv("ANALYZER_BINSTORE_BUCKETPREFIX", os.getenv("ANALYZER_BINARYSTORE_BUCKETPREFIX", "prj-")),
+    minioRegion=os.getenv("ANALYZER_BINSTORE_MINIO_REGION", os.getenv("ANALYZER_BINARYSTORE_MINIO_REGION", None)),
     instanceTaskType=os.getenv("INSTANCE_TASK_TYPE", "").strip(),
     filesystemDefaultPath=os.getenv("FILESYSTEM_DEFAULT_PATH", "storage").strip(),
     esChunkNumber=int(os.getenv("ES_CHUNK_NUMBER", "1000")),

test/__init__.py

@@ -46,7 +46,7 @@
     esClientKey='',
     appVersion='',
     minioRegion='',
-    minioBucketPrefix='',
+    bucketPrefix='',
     filesystemDefaultPath='',
     esChunkNumber=1000,
     binaryStoreType='filesystem',

test/mock_service.py

@@ -186,7 +186,7 @@ def setUp(self):
             esClientKey="",
             appVersion="",
             minioRegion="",
-            minioBucketPrefix="",
+            bucketPrefix="",
             filesystemDefaultPath="",
             esChunkNumber=1000,
             binaryStoreType="filesystem",

test/unit/commons/object_saving/test_filesystem_saver.py

@@ -128,9 +128,9 @@ def test_list_existing_folder():
     object_name = f'{random_alphanumeric(16)}.json'
     path = 'test'
     resource = '/'.join([path, object_name])
-    CREATED_FILES_AND_FOLDERS.append('/'.join([base_path, bucket, path, object_name]))
-    CREATED_FILES_AND_FOLDERS.append('/'.join([base_path, bucket, path]))
-    CREATED_FILES_AND_FOLDERS.append('/'.join([base_path, bucket]))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(base_path, f'prj-{bucket}', path, object_name))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(base_path, f'prj-{bucket}', path))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(base_path, f'prj-{bucket}'))
     CREATED_FILES_AND_FOLDERS.append(base_path)
 
     file_system = create_storage_client(base_path)
@@ -144,9 +144,9 @@ def test_list_dir_separators():
     object_name = f'{random_alphanumeric(16)}.json'
     path = 'test/'
     resource = path + object_name
-    CREATED_FILES_AND_FOLDERS.append('/'.join([bucket, path, object_name]))
-    CREATED_FILES_AND_FOLDERS.append('/'.join([bucket, path]))
-    CREATED_FILES_AND_FOLDERS.append('/'.join([bucket]))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(f'prj-{bucket}', path, object_name))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(f'prj-{bucket}', path))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(f'prj-{bucket}'))
 
     file_system = create_storage_client('')
     file_system.put_project_object({'test': True}, bucket, resource, using_json=True)
@@ -162,6 +162,8 @@ def test_remove_project_objects():
 
     file_system = create_storage_client('')
     file_system.put_project_object({'test': True}, bucket, resource, using_json=True)
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(f'prj-{bucket}', path))
+    CREATED_FILES_AND_FOLDERS.append(os.path.join(f'prj-{bucket}'))
 
     file_system.remove_project_objects(bucket, [resource])
     with pytest.raises(ValueError):

test/unit/commons/object_saving/test_minio_client.py

@@ -37,7 +37,7 @@ def run_s3():
 
 
 def create_storage_client(bucket_prefix=BUCKET_PREFIX):
-    return MinioClient(ApplicationConfig(minioHost=SERVER_HOST, minioRegion=REGION, minioBucketPrefix=bucket_prefix,
+    return MinioClient(ApplicationConfig(minioHost=SERVER_HOST, minioRegion=REGION, bucketPrefix=bucket_prefix,
                                          minioAccessKey='minio', minioSecretKey='minio', minioUseTls=False))