From 6f05a6de0ef6bef8d914c56a7831c8a9f61de87c Mon Sep 17 00:00:00 2001 From: dazarnov Date: Mon, 29 May 2023 18:22:50 +0300 Subject: [PATCH 1/6] Added bucket-name arg for RO commands --- medusa/download.py | 14 +++++----- medusa/fetch_tokenmap.py | 4 +-- medusa/listing.py | 4 +-- medusa/medusacli.py | 28 ++++++++++++------- medusa/restore_cluster.py | 6 ++-- medusa/restore_node.py | 8 +++--- medusa/status.py | 4 +-- medusa/storage/__init__.py | 17 +++++------ medusa/storage/abstract_storage.py | 5 ++-- medusa/storage/google_storage.py | 2 -- medusa/storage/local_storage.py | 3 +- medusa/storage/s3_base_storage.py | 2 -- .../features/steps/integration_steps.py | 4 +-- 13 files changed, 53 insertions(+), 48 deletions(-) diff --git a/medusa/download.py b/medusa/download.py index 3da5ece42..09dfc93cd 100644 --- a/medusa/download.py +++ b/medusa/download.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import json +import logging import pathlib import shutil import sys @@ -24,13 +24,12 @@ from medusa.filtering import filter_fqtns -def download_data(storageconfig, backup, fqtns_to_restore, destination): - +def download_data(storage_config, backup, fqtns_to_restore, destination): manifest = json.loads(backup.manifest) _check_available_space(manifest, destination) - with Storage(config=storageconfig) as storage: + with Storage(config=storage_config) as storage: for section in manifest: @@ -70,9 +69,10 @@ def download_data(storageconfig, backup, fqtns_to_restore, destination): ) -def download_cmd(config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces): - - with Storage(config=config.storage) as storage: +def download_cmd( + config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name=None +): + with Storage(config=config.storage, bucket_name=bucket_name) as storage: if not download_destination.is_dir(): logging.error('{} is not a directory'.format(download_destination)) diff --git a/medusa/fetch_tokenmap.py b/medusa/fetch_tokenmap.py index 26a92dc82..ba9d54fa8 100644 --- a/medusa/fetch_tokenmap.py +++ b/medusa/fetch_tokenmap.py @@ -18,8 +18,8 @@ from medusa.storage import Storage -def main(config, backup_name): - with Storage(config=config.storage) as storage: +def main(config, backup_name, bucket_name): + with Storage(config=config.storage, bucket_name=bucket_name) as storage: backup = storage.get_cluster_backup(backup_name) if not backup: logging.error('No such backup') diff --git a/medusa/listing.py b/medusa/listing.py index 2799e268b..17b186801 100644 --- a/medusa/listing.py +++ b/medusa/listing.py @@ -36,8 +36,8 @@ def get_backups(storage, config, show_all): return cluster_backups -def list_backups(config, show_all): - with Storage(config=config.storage) as storage: +def list_backups(config, show_all, bucket_name=None): + with Storage(config=config.storage, bucket_name=bucket_name) as storage: list_backups_w_storage(config, show_all, storage) diff --git a/medusa/medusacli.py b/medusa/medusacli.py index ff460fe6e..80a765206 100644 --- a/medusa/medusacli.py +++ b/medusa/medusacli.py @@ -167,22 +167,24 @@ def backup_cluster(medusaconfig, backup_name, seed_target, stagger, enable_md5_c @cli.command(name='fetch-tokenmap') @click.option('--backup-name', help='backup name', required=True) +@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) @pass_MedusaConfig -def fetch_tokenmap(medusaconfig, backup_name): +def fetch_tokenmap(medusaconfig, backup_name, bucket_name): """ Get the token/node mapping for a specific backup """ - medusa.fetch_tokenmap.main(medusaconfig, backup_name) + medusa.fetch_tokenmap.main(medusaconfig, backup_name, bucket_name) @cli.command(name='list-backups') @click.option('--show-all/--no-show-all', default=False, help="List all backups in the bucket") +@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) @pass_MedusaConfig -def list_backups(medusaconfig, show_all): +def list_backups(medusaconfig, show_all, bucket_name): """ List backups """ - medusa.listing.list_backups(medusaconfig, show_all) + medusa.listing.list_backups(medusaconfig, show_all, bucket_name) @cli.command(name='download') @@ -194,13 +196,14 @@ def list_backups(medusaconfig, show_all): multiple=True, default={}) @click.option('--ignore-system-keyspaces', help='Do not download cassandra system keyspaces', required=True, is_flag=True, default=False) +@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) @pass_MedusaConfig -def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces): +def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name): """ Download backup """ medusa.download.download_cmd(medusaconfig, backup_name, Path(download_destination), keyspaces, tables, - ignore_system_keyspaces) + ignore_system_keyspaces, bucket_name) @cli.command(name='restore-cluster') @@ -224,9 +227,11 @@ def download(medusaconfig, backup_name, download_destination, keyspaces, tables, @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") @click.option('--ignore-racks', help='Disable matching nodes based on rack topology', required=False, default=False, is_flag=True) +@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) @pass_MedusaConfig def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, - verify, keyspaces, tables, parallel_restores, use_sstableloader, version_target, ignore_racks): + verify, keyspaces, tables, parallel_restores, use_sstableloader, version_target, ignore_racks, + bucket_name): """ Restore Cassandra cluster """ @@ -243,7 +248,8 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, int(parallel_restores), use_sstableloader, version_target, - ignore_racks) + ignore_racks, + bucket_name) @cli.command(name='restore-node') @@ -264,14 +270,16 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, @click.option('--use-sstableloader', help='Use the sstableloader to load the backup into the cluster', default=False, is_flag=True) @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") +@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) @pass_MedusaConfig def restore_node(medusaconfig, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, - use_sstableloader, version_target): + use_sstableloader, version_target, bucket_name): """ Restore single Cassandra node """ medusa.restore_node.restore_node(medusaconfig, Path(temp_dir), backup_name, in_place, keep_auth, seeds, - verify, set(keyspaces), set(tables), use_sstableloader, version_target) + verify, set(keyspaces), set(tables), use_sstableloader, version_target, + bucket_name) @cli.command(name='status') diff --git a/medusa/restore_cluster.py b/medusa/restore_cluster.py index 9a85b358e..24163800e 100644 --- a/medusa/restore_cluster.py +++ b/medusa/restore_cluster.py @@ -35,7 +35,8 @@ def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, verify, keyspaces, - tables, parallel_restores, use_sstableloader=False, version_target=None, ignore_racks=False): + tables, parallel_restores, use_sstableloader=False, version_target=None, ignore_racks=False, + bucket_name=None): monitoring = Monitoring(config=config.monitoring) try: restore_start_time = datetime.datetime.now() @@ -57,8 +58,7 @@ def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth logging.error(err_msg) raise RuntimeError(err_msg) - with Storage(config=config.storage) as storage: - + with Storage(config=config.storage, bucket_name=bucket_name) as storage: try: cluster_backup = storage.get_cluster_backup(backup_name) except KeyError: diff --git a/medusa/restore_node.py b/medusa/restore_node.py index e0a3a0b52..f617cd425 100644 --- a/medusa/restore_node.py +++ b/medusa/restore_node.py @@ -38,12 +38,12 @@ def restore_node(config, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, - use_sstableloader=False, version_target=None): + use_sstableloader=False, version_target=None, bucket_name=None): if in_place and keep_auth: logging.error('Cannot keep system_auth when restoring in-place. It would be overwritten') sys.exit(1) - with Storage(config=config.storage) as storage: + with Storage(config=config.storage, bucket_name=bucket_name) as storage: capture_release_version(storage, version_target) if not use_sstableloader: @@ -87,7 +87,7 @@ def restore_node_locally(config, temp_dir, backup_name, in_place, keep_auth, see # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) - download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) + download_data(storage, node_backup, fqtns_to_restore, destination=download_dir) if not medusa.utils.evaluate_boolean(config.kubernetes.enabled if config.kubernetes else False): logging.info('Stopping Cassandra') @@ -184,7 +184,7 @@ def restore_node_sstableloader(config, temp_dir, backup_name, in_place, keep_aut # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) - download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) + download_data(storage, node_backup, fqtns_to_restore, destination=download_dir) invoke_sstableloader(config, download_dir, keep_auth, fqtns_to_restore, cassandra.storage_port, cassandra.native_port) logging.info('Finished loading backup from {}'.format(fqdn)) diff --git a/medusa/status.py b/medusa/status.py index eb1fe5f10..9c60088a2 100644 --- a/medusa/status.py +++ b/medusa/status.py @@ -23,8 +23,8 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' -def status(config, backup_name): - with Storage(config=config.storage) as storage: +def status(config, backup_name, bucket_name=None): + storage = Storage(config=config.storage, bucket_name=bucket_name) try: cluster_backup = storage.get_cluster_backup(backup_name) diff --git a/medusa/storage/__init__.py b/medusa/storage/__init__.py index 73ffc1ce7..626886f51 100644 --- a/medusa/storage/__init__.py +++ b/medusa/storage/__init__.py @@ -62,7 +62,7 @@ def format_bytes_str(value): class Storage(object): - def __init__(self, *, config): + def __init__(self, *, config, bucket_name=None): self._config = config # Used to bypass dependency checks when running in Kubernetes self._k8s_mode = evaluate_boolean(config.k8s_mode) if config.k8s_mode else False @@ -70,6 +70,7 @@ def __init__(self, *, config): self.prefix_path = str(self._prefix) + '/' if len(str(self._prefix)) > 1 else '' self.storage_driver = self._load_storage() self.storage_provider = self._config.storage_provider + self._bucket_name = bucket_name def __enter__(self): self.storage_driver.connect() @@ -81,23 +82,23 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _load_storage(self): logging.debug('Loading storage_provider: {}'.format(self._config.storage_provider)) if self._config.storage_provider.lower() == 'google_storage': - google_storage = GoogleStorage(self._config) + google_storage = GoogleStorage(self._config, bucket_name=self._bucket_name) return google_storage elif self._config.storage_provider.lower() == 'azure_blobs': - azure_storage = AzureStorage(self._config) + azure_storage = AzureStorage(self._config, bucket_name=self._bucket_name) return azure_storage elif self._config.storage_provider.lower() == 's3_rgw': - return S3RGWStorage(self._config) + return S3RGWStorage(self._config, bucket_name=self._bucket_name) elif self._config.storage_provider.lower() == "s3_compatible": - s3_storage = S3BaseStorage(self._config) + s3_storage = S3BaseStorage(self._config, bucket_name=self._bucket_name) return s3_storage elif self._config.storage_provider.lower().startswith('s3'): - s3_storage = S3Storage(self._config) + s3_storage = S3Storage(self._config, bucket_name=self._bucket_name) return s3_storage elif self._config.storage_provider.lower() == 'local': - return LocalStorage(self._config) + return LocalStorage(self._config, bucket_name=self._bucket_name) elif self._config.storage_provider.lower() == "ibm_storage": - s3_storage = S3BaseStorage(self._config) + s3_storage = S3BaseStorage(self._config, bucket_name=self._bucket_name) return s3_storage raise NotImplementedError("Unsupported storage provider") diff --git a/medusa/storage/abstract_storage.py b/medusa/storage/abstract_storage.py index d21aa22c7..734d33c41 100644 --- a/medusa/storage/abstract_storage.py +++ b/medusa/storage/abstract_storage.py @@ -52,9 +52,10 @@ class AbstractStorage(abc.ABC): # sometimes we store Cassandra version in this it seems api_version = None - def __init__(self, config): + def __init__(self, config, bucket_name=None): self.config = config - self.bucket_name = config.bucket_name + self.bucket_name = bucket_name if bucket_name is not None else config.bucket_name + logging.debug(f'Using bucket {self.bucket_name}') @abc.abstractmethod def connect(self): diff --git a/medusa/storage/google_storage.py b/medusa/storage/google_storage.py index 01fc2863d..7622ef93f 100644 --- a/medusa/storage/google_storage.py +++ b/medusa/storage/google_storage.py @@ -43,8 +43,6 @@ def __init__(self, config): self.service_file = str(Path(config.key_file).expanduser()) logging.info("Using service file: {}".format(self.service_file)) - self.bucket_name = config.bucket_name - logging.debug('Connecting to Google Storage') logging.getLogger('gcloud.aio.storage.storage').setLevel(logging.WARNING) diff --git a/medusa/storage/local_storage.py b/medusa/storage/local_storage.py index 41e0f7c7f..a15481bb5 100644 --- a/medusa/storage/local_storage.py +++ b/medusa/storage/local_storage.py @@ -32,7 +32,6 @@ class LocalStorage(AbstractStorage): def __init__(self, config): self.config = config - self.bucket_name = self.config.bucket_name self.root_dir = Path(config.base_path) / self.bucket_name self.root_dir.mkdir(parents=True, exist_ok=True) @@ -177,7 +176,7 @@ def get_object_datetime(self, blob): def get_cache_path(self, path): # Full path for files that will be taken from previous backups - return "{}/{}/{}".format(self.config.base_path, self.config.bucket_name, path) + return "{}/{}/{}".format(self.config.base_path, self.bucket_name, path) @staticmethod def blob_matches_manifest(blob, object_in_manifest, enable_md5_checks=False): diff --git a/medusa/storage/s3_base_storage.py b/medusa/storage/s3_base_storage.py index 2c250f859..04cc348f8 100644 --- a/medusa/storage/s3_base_storage.py +++ b/medusa/storage/s3_base_storage.py @@ -110,8 +110,6 @@ def __init__(self, config): self.credentials = self._consolidate_credentials(config) logging.info('Using credentials {}'.format(self.credentials)) - self.bucket_name: str = config.bucket_name - self.storage_provider = config.storage_provider self.connection_extra_args = self._make_connection_arguments(config) diff --git a/tests/integration/features/steps/integration_steps.py b/tests/integration/features/steps/integration_steps.py index f7f45ff9b..b3e269679 100644 --- a/tests/integration/features/steps/integration_steps.py +++ b/tests/integration/features/steps/integration_steps.py @@ -1214,7 +1214,7 @@ def _there_is_no_latest_complete_backup(context): @then(r"I can list and print backups without errors") def _can_list_print_backups_without_error(context): - medusa.listing.list_backups(config=context.medusa_config, show_all=True) + medusa.listing.list_backups(config=context.medusa_config, show_all=True, bucket_name=None) @then(r'the latest complete cluster backup is "{expected_backup_name}"') @@ -1491,7 +1491,7 @@ def _i_delete_the_backup_named(context, backup_name, all_nodes=False): @then(r'I can fetch the tokenmap of the backup named "{backup_name}"') def _i_can_fecth_tokenmap_of_backup_named(context, backup_name): - tokenmap = medusa.fetch_tokenmap.main(backup_name=backup_name, config=context.medusa_config) + tokenmap = medusa.fetch_tokenmap.main(backup_name=backup_name, config=context.medusa_config, bucket_name=None) assert "127.0.0.1" in tokenmap From 08d4c6b7adb8b6b3fd8dbee943bf604d7207262c Mon Sep 17 00:00:00 2001 From: dazarnov Date: Mon, 29 May 2023 18:45:15 +0300 Subject: [PATCH 2/6] Added bucket-name arg for RO commands --- medusa/restore_cluster.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/medusa/restore_cluster.py b/medusa/restore_cluster.py index 24163800e..3bd077a19 100644 --- a/medusa/restore_cluster.py +++ b/medusa/restore_cluster.py @@ -68,7 +68,7 @@ def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth restore = RestoreJob(cluster_backup, config, temp_dir, host_list, seed_target, keep_auth, verify, parallel_restores, keyspaces, tables, bypass_checks, use_sstableloader, version_target, - ignore_racks) + ignore_racks, bucket_name) restore.execute() restore_end_time = datetime.datetime.now() @@ -103,7 +103,7 @@ class RestoreJob(object): def __init__(self, cluster_backup, config, temp_dir, host_list, seed_target, keep_auth, verify, parallel_restores, keyspaces=None, tables=None, bypass_checks=False, use_sstableloader=False, - version_target=None, ignore_racks=False): + version_target=None, ignore_racks=False, bucket_name=None): self.id = uuid.uuid4() self.ringmap = None self.cluster_backup = cluster_backup @@ -129,6 +129,7 @@ def __init__(self, cluster_backup, config, temp_dir, host_list, seed_target, kee self.fqdn_resolver = HostnameResolver(fqdn_resolver, k8s_mode) self._version_target = version_target self.ignore_racks = ignore_racks + self.bucket_name = bucket_name def prepare_restore(self): logging.info('Ensuring the backup is found and is complete') @@ -427,7 +428,7 @@ def _build_restore_cmd(self): command = 'mkdir -p {work}; cd {work} && medusa-wrapper {sudo} medusa {config} ' \ '--fqdn=%s -vvv restore-node ' \ '{in_place} {keep_auth} %s {verify} --backup-name {backup} --temp-dir {temp_dir} ' \ - '{use_sstableloader} {keyspaces} {tables}' \ + '{use_sstableloader} {keyspaces} {tables} {bucket_name}' \ .format(work=self.work_dir, sudo='sudo' if medusa.utils.evaluate_boolean(self.config.cassandra.use_sudo) else '', config=f'--config-file {self.config.file_path}' if self.config.file_path else '', @@ -438,7 +439,8 @@ def _build_restore_cmd(self): temp_dir=self.temp_dir, use_sstableloader='--use-sstableloader' if self.use_sstableloader else '', keyspaces=keyspace_options, - tables=table_options) + tables=table_options, + bucket_name=f'--bucket-name {self.bucket_name}' if self.bucket_name is not None else '') logging.debug('Preparing to restore on all nodes with the following command: {}'.format(command)) From aca6a48c71d21244ea1edd119e84fd48ee243216 Mon Sep 17 00:00:00 2001 From: dazarnov Date: Mon, 29 May 2023 18:53:46 +0300 Subject: [PATCH 3/6] Added bucket-name arg for RO commands --- medusa/storage/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/medusa/storage/__init__.py b/medusa/storage/__init__.py index 626886f51..d6f3c0e3d 100644 --- a/medusa/storage/__init__.py +++ b/medusa/storage/__init__.py @@ -27,14 +27,16 @@ from medusa.storage.cluster_backup import ClusterBackup from medusa.storage.node_backup import NodeBackup from medusa.storage.abstract_storage import ManifestObject, AbstractBlob +from medusa.storage.azure_storage import AzureStorage +from medusa.storage.cluster_backup import ClusterBackup from medusa.storage.google_storage import GoogleStorage from medusa.storage.local_storage import LocalStorage -from medusa.storage.s3_storage import S3Storage -from medusa.storage.s3_rgw import S3RGWStorage -from medusa.storage.azure_storage import AzureStorage +from medusa.storage.node_backup import NodeBackup from medusa.storage.s3_base_storage import S3BaseStorage -from medusa.utils import evaluate_boolean +from medusa.storage.s3_rgw import S3RGWStorage +from medusa.storage.s3_storage import S3Storage +from medusa.utils import evaluate_boolean # pattern meant to match just the blob name, not the entire path # the path is covered by the initial .* @@ -68,9 +70,10 @@ def __init__(self, *, config, bucket_name=None): self._k8s_mode = evaluate_boolean(config.k8s_mode) if config.k8s_mode else False self._prefix = pathlib.Path(config.prefix or '.') self.prefix_path = str(self._prefix) + '/' if len(str(self._prefix)) > 1 else '' + self._bucket_name = bucket_name self.storage_driver = self._load_storage() - self.storage_provider = self._config.storage_provider self._bucket_name = bucket_name + self.storage_provider = self._config.storage_provider def __enter__(self): self.storage_driver.connect() From 7af1b8766be1d34706744373738d32f79afc6ac1 Mon Sep 17 00:00:00 2001 From: dazarnov Date: Wed, 28 Jun 2023 20:18:57 +0300 Subject: [PATCH 4/6] Added prefix arg for RO commands --- medusa/download.py | 5 ++-- medusa/fetch_tokenmap.py | 4 +-- medusa/listing.py | 9 ++++--- medusa/medusacli.py | 27 ++++++++++++------- medusa/restore_cluster.py | 15 ++++++----- medusa/restore_node.py | 4 +-- medusa/storage/__init__.py | 5 ++-- .../features/steps/integration_steps.py | 5 ++-- 8 files changed, 42 insertions(+), 32 deletions(-) diff --git a/medusa/download.py b/medusa/download.py index 09dfc93cd..5f4bc6fe3 100644 --- a/medusa/download.py +++ b/medusa/download.py @@ -69,9 +69,8 @@ def download_data(storage_config, backup, fqtns_to_restore, destination): ) -def download_cmd( - config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name=None -): +def download_cmd(config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, + bucket_name=None, prefix=None): with Storage(config=config.storage, bucket_name=bucket_name) as storage: if not download_destination.is_dir(): diff --git a/medusa/fetch_tokenmap.py b/medusa/fetch_tokenmap.py index ba9d54fa8..c23047a39 100644 --- a/medusa/fetch_tokenmap.py +++ b/medusa/fetch_tokenmap.py @@ -18,8 +18,8 @@ from medusa.storage import Storage -def main(config, backup_name, bucket_name): - with Storage(config=config.storage, bucket_name=bucket_name) as storage: +def main(config, backup_name, bucket_name, prefix): + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: backup = storage.get_cluster_backup(backup_name) if not backup: logging.error('No such backup') diff --git a/medusa/listing.py b/medusa/listing.py index 17b186801..c6a8d43bf 100644 --- a/medusa/listing.py +++ b/medusa/listing.py @@ -21,7 +21,8 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' -def get_backups(storage, config, show_all): +def get_backups(config, show_all): + storage = Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) cluster_backups = sorted( storage.list_cluster_backups(), @@ -36,13 +37,13 @@ def get_backups(storage, config, show_all): return cluster_backups -def list_backups(config, show_all, bucket_name=None): - with Storage(config=config.storage, bucket_name=bucket_name) as storage: +def list_backups(config, show_all, bucket_name=None, prefix=None): + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: list_backups_w_storage(config, show_all, storage) def list_backups_w_storage(config, show_all, storage): - cluster_backups = get_backups(storage, config, show_all) + cluster_backups = get_backups(config, show_all) seen_incomplete_backup = False for cluster_backup in cluster_backups: finished = cluster_backup.finished diff --git a/medusa/medusacli.py b/medusa/medusacli.py index 80a765206..500fe010c 100644 --- a/medusa/medusacli.py +++ b/medusa/medusacli.py @@ -168,23 +168,25 @@ def backup_cluster(medusaconfig, backup_name, seed_target, stagger, enable_md5_c @cli.command(name='fetch-tokenmap') @click.option('--backup-name', help='backup name', required=True) @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) +@click.option('--prefix', help='Backup prefix', required=False, default=None) @pass_MedusaConfig -def fetch_tokenmap(medusaconfig, backup_name, bucket_name): +def fetch_tokenmap(medusaconfig, backup_name, bucket_name, prefix): """ Get the token/node mapping for a specific backup """ - medusa.fetch_tokenmap.main(medusaconfig, backup_name, bucket_name) + medusa.fetch_tokenmap.main(medusaconfig, backup_name, bucket_name, prefix) @cli.command(name='list-backups') @click.option('--show-all/--no-show-all', default=False, help="List all backups in the bucket") @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) +@click.option('--prefix', help='Backup prefix', required=False, default=None) @pass_MedusaConfig -def list_backups(medusaconfig, show_all, bucket_name): +def list_backups(medusaconfig, show_all, bucket_name, prefix): """ List backups """ - medusa.listing.list_backups(medusaconfig, show_all, bucket_name) + medusa.listing.list_backups(medusaconfig, show_all, bucket_name, prefix) @cli.command(name='download') @@ -197,13 +199,15 @@ def list_backups(medusaconfig, show_all, bucket_name): @click.option('--ignore-system-keyspaces', help='Do not download cassandra system keyspaces', required=True, is_flag=True, default=False) @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) +@click.option('--prefix', help='Backup prefix', required=False, default=None) @pass_MedusaConfig -def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name): +def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name, + prefix): """ Download backup """ medusa.download.download_cmd(medusaconfig, backup_name, Path(download_destination), keyspaces, tables, - ignore_system_keyspaces, bucket_name) + ignore_system_keyspaces, bucket_name, prefix) @cli.command(name='restore-cluster') @@ -228,10 +232,11 @@ def download(medusaconfig, backup_name, download_destination, keyspaces, tables, @click.option('--ignore-racks', help='Disable matching nodes based on rack topology', required=False, default=False, is_flag=True) @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) +@click.option('--prefix', help='Backup prefix', required=False, default=None) @pass_MedusaConfig def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, verify, keyspaces, tables, parallel_restores, use_sstableloader, version_target, ignore_racks, - bucket_name): + bucket_name, prefix): """ Restore Cassandra cluster """ @@ -249,7 +254,8 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, use_sstableloader, version_target, ignore_racks, - bucket_name) + bucket_name, + prefix) @cli.command(name='restore-node') @@ -271,15 +277,16 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, default=False, is_flag=True) @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) +@click.option('--prefix', help='Backup prefix', required=False, default=None) @pass_MedusaConfig def restore_node(medusaconfig, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, - use_sstableloader, version_target, bucket_name): + use_sstableloader, version_target, bucket_name, prefix): """ Restore single Cassandra node """ medusa.restore_node.restore_node(medusaconfig, Path(temp_dir), backup_name, in_place, keep_auth, seeds, verify, set(keyspaces), set(tables), use_sstableloader, version_target, - bucket_name) + bucket_name, prefix) @cli.command(name='status') diff --git a/medusa/restore_cluster.py b/medusa/restore_cluster.py index 3bd077a19..31fa6c8f9 100644 --- a/medusa/restore_cluster.py +++ b/medusa/restore_cluster.py @@ -36,7 +36,7 @@ def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, verify, keyspaces, tables, parallel_restores, use_sstableloader=False, version_target=None, ignore_racks=False, - bucket_name=None): + bucket_name=None, prefix=None): monitoring = Monitoring(config=config.monitoring) try: restore_start_time = datetime.datetime.now() @@ -58,17 +58,16 @@ def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth logging.error(err_msg) raise RuntimeError(err_msg) - with Storage(config=config.storage, bucket_name=bucket_name) as storage: + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: try: cluster_backup = storage.get_cluster_backup(backup_name) except KeyError: err_msg = 'No such backup --> {}'.format(backup_name) logging.error(err_msg) raise RuntimeError(err_msg) - restore = RestoreJob(cluster_backup, config, temp_dir, host_list, seed_target, keep_auth, verify, parallel_restores, keyspaces, tables, bypass_checks, use_sstableloader, version_target, - ignore_racks, bucket_name) + ignore_racks, bucket_name, prefix) restore.execute() restore_end_time = datetime.datetime.now() @@ -103,7 +102,7 @@ class RestoreJob(object): def __init__(self, cluster_backup, config, temp_dir, host_list, seed_target, keep_auth, verify, parallel_restores, keyspaces=None, tables=None, bypass_checks=False, use_sstableloader=False, - version_target=None, ignore_racks=False, bucket_name=None): + version_target=None, ignore_racks=False, bucket_name=None, prefix=None): self.id = uuid.uuid4() self.ringmap = None self.cluster_backup = cluster_backup @@ -130,6 +129,7 @@ def __init__(self, cluster_backup, config, temp_dir, host_list, seed_target, kee self._version_target = version_target self.ignore_racks = ignore_racks self.bucket_name = bucket_name + self.prefix = prefix def prepare_restore(self): logging.info('Ensuring the backup is found and is complete') @@ -428,7 +428,7 @@ def _build_restore_cmd(self): command = 'mkdir -p {work}; cd {work} && medusa-wrapper {sudo} medusa {config} ' \ '--fqdn=%s -vvv restore-node ' \ '{in_place} {keep_auth} %s {verify} --backup-name {backup} --temp-dir {temp_dir} ' \ - '{use_sstableloader} {keyspaces} {tables} {bucket_name}' \ + '{use_sstableloader} {keyspaces} {tables} {bucket_name} {prefix}' \ .format(work=self.work_dir, sudo='sudo' if medusa.utils.evaluate_boolean(self.config.cassandra.use_sudo) else '', config=f'--config-file {self.config.file_path}' if self.config.file_path else '', @@ -440,7 +440,8 @@ def _build_restore_cmd(self): use_sstableloader='--use-sstableloader' if self.use_sstableloader else '', keyspaces=keyspace_options, tables=table_options, - bucket_name=f'--bucket-name {self.bucket_name}' if self.bucket_name is not None else '') + bucket_name=f'--bucket-name {self.bucket_name}' if self.bucket_name is not None else '', + prefix=f'--prefix {self.prefix}' if self.prefix is not None else '') logging.debug('Preparing to restore on all nodes with the following command: {}'.format(command)) diff --git a/medusa/restore_node.py b/medusa/restore_node.py index f617cd425..4b2890798 100644 --- a/medusa/restore_node.py +++ b/medusa/restore_node.py @@ -38,12 +38,12 @@ def restore_node(config, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, - use_sstableloader=False, version_target=None, bucket_name=None): + use_sstableloader=False, version_target=None, bucket_name=None, prefix=None): if in_place and keep_auth: logging.error('Cannot keep system_auth when restoring in-place. It would be overwritten') sys.exit(1) - with Storage(config=config.storage, bucket_name=bucket_name) as storage: + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: capture_release_version(storage, version_target) if not use_sstableloader: diff --git a/medusa/storage/__init__.py b/medusa/storage/__init__.py index d6f3c0e3d..18fbcdd84 100644 --- a/medusa/storage/__init__.py +++ b/medusa/storage/__init__.py @@ -64,11 +64,12 @@ def format_bytes_str(value): class Storage(object): - def __init__(self, *, config, bucket_name=None): + def __init__(self, *, config, bucket_name=None, prefix=None): self._config = config # Used to bypass dependency checks when running in Kubernetes self._k8s_mode = evaluate_boolean(config.k8s_mode) if config.k8s_mode else False - self._prefix = pathlib.Path(config.prefix or '.') + self._prefix = pathlib.Path(prefix or config.prefix or '.') + logging.debug(f'Using prefix {self._prefix}') self.prefix_path = str(self._prefix) + '/' if len(str(self._prefix)) > 1 else '' self._bucket_name = bucket_name self.storage_driver = self._load_storage() diff --git a/tests/integration/features/steps/integration_steps.py b/tests/integration/features/steps/integration_steps.py index b3e269679..53af14444 100644 --- a/tests/integration/features/steps/integration_steps.py +++ b/tests/integration/features/steps/integration_steps.py @@ -1214,7 +1214,7 @@ def _there_is_no_latest_complete_backup(context): @then(r"I can list and print backups without errors") def _can_list_print_backups_without_error(context): - medusa.listing.list_backups(config=context.medusa_config, show_all=True, bucket_name=None) + medusa.listing.list_backups(config=context.medusa_config, show_all=True, bucket_name=None, prefix=None) @then(r'the latest complete cluster backup is "{expected_backup_name}"') @@ -1491,7 +1491,8 @@ def _i_delete_the_backup_named(context, backup_name, all_nodes=False): @then(r'I can fetch the tokenmap of the backup named "{backup_name}"') def _i_can_fecth_tokenmap_of_backup_named(context, backup_name): - tokenmap = medusa.fetch_tokenmap.main(backup_name=backup_name, config=context.medusa_config, bucket_name=None) + tokenmap = medusa.fetch_tokenmap.main(backup_name=backup_name, config=context.medusa_config, bucket_name=None, + prefix=None) assert "127.0.0.1" in tokenmap From 56b1d0739bf7d6c684a6ada7320f90dbdc7c64ad Mon Sep 17 00:00:00 2001 From: Radovan Zvoncek Date: Tue, 23 Apr 2024 13:42:53 +0200 Subject: [PATCH 5/6] Fix post-rebase bugs --- medusa/download.py | 2 +- medusa/listing.py | 6 ++---- medusa/medusacli.py | 20 ++++++++++---------- medusa/restore_node.py | 4 ++-- medusa/status.py | 4 ++-- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/medusa/download.py b/medusa/download.py index 5f4bc6fe3..768e18a6d 100644 --- a/medusa/download.py +++ b/medusa/download.py @@ -71,7 +71,7 @@ def download_data(storage_config, backup, fqtns_to_restore, destination): def download_cmd(config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name=None, prefix=None): - with Storage(config=config.storage, bucket_name=bucket_name) as storage: + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: if not download_destination.is_dir(): logging.error('{} is not a directory'.format(download_destination)) diff --git a/medusa/listing.py b/medusa/listing.py index c6a8d43bf..a93e2b156 100644 --- a/medusa/listing.py +++ b/medusa/listing.py @@ -21,9 +21,7 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' -def get_backups(config, show_all): - storage = Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) - +def get_backups(storage, config, show_all): cluster_backups = sorted( storage.list_cluster_backups(), key=lambda b: b.started @@ -43,7 +41,7 @@ def list_backups(config, show_all, bucket_name=None, prefix=None): def list_backups_w_storage(config, show_all, storage): - cluster_backups = get_backups(config, show_all) + cluster_backups = get_backups(storage, config, show_all) seen_incomplete_backup = False for cluster_backup in cluster_backups: finished = cluster_backup.finished diff --git a/medusa/medusacli.py b/medusa/medusacli.py index 500fe010c..a02c55b60 100644 --- a/medusa/medusacli.py +++ b/medusa/medusacli.py @@ -98,7 +98,7 @@ def configure_console_logging(verbosity, without_log_timestamp): @click.option('-v', '--verbosity', help='Verbosity', default=0, count=True) @click.option('--without-log-timestamp', help='Do not show timestamp in logs', default=False, is_flag=True) @click.option('--config-file', help='Specify config file') -@click.option('--bucket-name', help='Bucket name') +@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config') @click.option('--key-file', help='GCP credentials key file') @click.option('--prefix', help='Prefix for shared storage') @click.option('--fqdn', help='Act as another host') @@ -168,7 +168,7 @@ def backup_cluster(medusaconfig, backup_name, seed_target, stagger, enable_md5_c @cli.command(name='fetch-tokenmap') @click.option('--backup-name', help='backup name', required=True) @click.option('--bucket-name', help='Bucket with backup', required=False, default=None) -@click.option('--prefix', help='Backup prefix', required=False, default=None) +@click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def fetch_tokenmap(medusaconfig, backup_name, bucket_name, prefix): """ @@ -179,8 +179,8 @@ def fetch_tokenmap(medusaconfig, backup_name, bucket_name, prefix): @cli.command(name='list-backups') @click.option('--show-all/--no-show-all', default=False, help="List all backups in the bucket") -@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) -@click.option('--prefix', help='Backup prefix', required=False, default=None) +@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def list_backups(medusaconfig, show_all, bucket_name, prefix): """ @@ -198,8 +198,8 @@ def list_backups(medusaconfig, show_all, bucket_name, prefix): multiple=True, default={}) @click.option('--ignore-system-keyspaces', help='Do not download cassandra system keyspaces', required=True, is_flag=True, default=False) -@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) -@click.option('--prefix', help='Backup prefix', required=False, default=None) +@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name, prefix): @@ -231,8 +231,8 @@ def download(medusaconfig, backup_name, download_destination, keyspaces, tables, @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") @click.option('--ignore-racks', help='Disable matching nodes based on rack topology', required=False, default=False, is_flag=True) -@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) -@click.option('--prefix', help='Backup prefix', required=False, default=None) +@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, verify, keyspaces, tables, parallel_restores, use_sstableloader, version_target, ignore_racks, @@ -276,8 +276,8 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, @click.option('--use-sstableloader', help='Use the sstableloader to load the backup into the cluster', default=False, is_flag=True) @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") -@click.option('--bucket-name', help='Bucket with backup', required=False, default=None) -@click.option('--prefix', help='Backup prefix', required=False, default=None) +@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def restore_node(medusaconfig, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, use_sstableloader, version_target, bucket_name, prefix): diff --git a/medusa/restore_node.py b/medusa/restore_node.py index 4b2890798..5f8dd2d14 100644 --- a/medusa/restore_node.py +++ b/medusa/restore_node.py @@ -87,7 +87,7 @@ def restore_node_locally(config, temp_dir, backup_name, in_place, keep_auth, see # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) - download_data(storage, node_backup, fqtns_to_restore, destination=download_dir) + download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) if not medusa.utils.evaluate_boolean(config.kubernetes.enabled if config.kubernetes else False): logging.info('Stopping Cassandra') @@ -184,7 +184,7 @@ def restore_node_sstableloader(config, temp_dir, backup_name, in_place, keep_aut # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) - download_data(storage, node_backup, fqtns_to_restore, destination=download_dir) + download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) invoke_sstableloader(config, download_dir, keep_auth, fqtns_to_restore, cassandra.storage_port, cassandra.native_port) logging.info('Finished loading backup from {}'.format(fqdn)) diff --git a/medusa/status.py b/medusa/status.py index 9c60088a2..6283ba439 100644 --- a/medusa/status.py +++ b/medusa/status.py @@ -23,8 +23,8 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' -def status(config, backup_name, bucket_name=None): - storage = Storage(config=config.storage, bucket_name=bucket_name) +def status(config, backup_name, bucket_name=None, prefix=None): + with Storage(config=config.storage, bucket_name=bucket_name, prefix=prefix) as storage: try: cluster_backup = storage.get_cluster_backup(backup_name) From 9e3e95542f5d4f4c748f6fc71bbb527ab85fe1ec Mon Sep 17 00:00:00 2001 From: Radovan Zvoncek Date: Tue, 23 Apr 2024 13:46:58 +0200 Subject: [PATCH 6/6] Fix flake and unit test errors --- medusa/medusacli.py | 12 ++++++++---- medusa/storage/__init__.py | 20 ++++++++------------ medusa/storage/local_storage.py | 5 +++-- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/medusa/medusacli.py b/medusa/medusacli.py index a02c55b60..e38d34cfb 100644 --- a/medusa/medusacli.py +++ b/medusa/medusacli.py @@ -179,7 +179,8 @@ def fetch_tokenmap(medusaconfig, backup_name, bucket_name, prefix): @cli.command(name='list-backups') @click.option('--show-all/--no-show-all', default=False, help="List all backups in the bucket") -@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--bucket-name', + help='Explicit bucket name to over-ride the one from config', required=False, default=None) @click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def list_backups(medusaconfig, show_all, bucket_name, prefix): @@ -198,7 +199,8 @@ def list_backups(medusaconfig, show_all, bucket_name, prefix): multiple=True, default={}) @click.option('--ignore-system-keyspaces', help='Do not download cassandra system keyspaces', required=True, is_flag=True, default=False) -@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--bucket-name', + help='Explicit bucket name to over-ride the one from config', required=False, default=None) @click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def download(medusaconfig, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces, bucket_name, @@ -231,7 +233,8 @@ def download(medusaconfig, backup_name, download_destination, keyspaces, tables, @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") @click.option('--ignore-racks', help='Disable matching nodes based on rack topology', required=False, default=False, is_flag=True) -@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--bucket-name', + help='Explicit bucket name to over-ride the one from config', required=False, default=None) @click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks, @@ -276,7 +279,8 @@ def restore_cluster(medusaconfig, backup_name, seed_target, temp_dir, host_list, @click.option('--use-sstableloader', help='Use the sstableloader to load the backup into the cluster', default=False, is_flag=True) @click.option('--version-target', help='Target Cassandra version', required=False, default="3.11.9") -@click.option('--bucket-name', help='Explicit bucket name to over-ride the one from config', required=False, default=None) +@click.option('--bucket-name', + help='Explicit bucket name to over-ride the one from config', required=False, default=None) @click.option('--prefix', help='Path prefix in multi-tenant buckets', required=False, default=None) @pass_MedusaConfig def restore_node(medusaconfig, temp_dir, backup_name, in_place, keep_auth, seeds, verify, keyspaces, tables, diff --git a/medusa/storage/__init__.py b/medusa/storage/__init__.py index 18fbcdd84..1a96c82af 100644 --- a/medusa/storage/__init__.py +++ b/medusa/storage/__init__.py @@ -24,18 +24,14 @@ import medusa.index -from medusa.storage.cluster_backup import ClusterBackup -from medusa.storage.node_backup import NodeBackup from medusa.storage.abstract_storage import ManifestObject, AbstractBlob from medusa.storage.azure_storage import AzureStorage from medusa.storage.cluster_backup import ClusterBackup from medusa.storage.google_storage import GoogleStorage from medusa.storage.local_storage import LocalStorage from medusa.storage.node_backup import NodeBackup -from medusa.storage.s3_base_storage import S3BaseStorage - from medusa.storage.s3_rgw import S3RGWStorage -from medusa.storage.s3_storage import S3Storage +from medusa.storage.s3_storage import S3BaseStorage, S3Storage from medusa.utils import evaluate_boolean # pattern meant to match just the blob name, not the entire path @@ -86,23 +82,23 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _load_storage(self): logging.debug('Loading storage_provider: {}'.format(self._config.storage_provider)) if self._config.storage_provider.lower() == 'google_storage': - google_storage = GoogleStorage(self._config, bucket_name=self._bucket_name) + google_storage = GoogleStorage(self._config) return google_storage elif self._config.storage_provider.lower() == 'azure_blobs': - azure_storage = AzureStorage(self._config, bucket_name=self._bucket_name) + azure_storage = AzureStorage(self._config) return azure_storage elif self._config.storage_provider.lower() == 's3_rgw': - return S3RGWStorage(self._config, bucket_name=self._bucket_name) + return S3RGWStorage(self._config) elif self._config.storage_provider.lower() == "s3_compatible": - s3_storage = S3BaseStorage(self._config, bucket_name=self._bucket_name) + s3_storage = S3BaseStorage(self._config) return s3_storage elif self._config.storage_provider.lower().startswith('s3'): - s3_storage = S3Storage(self._config, bucket_name=self._bucket_name) + s3_storage = S3Storage(self._config) return s3_storage elif self._config.storage_provider.lower() == 'local': - return LocalStorage(self._config, bucket_name=self._bucket_name) + return LocalStorage(self._config) elif self._config.storage_provider.lower() == "ibm_storage": - s3_storage = S3BaseStorage(self._config, bucket_name=self._bucket_name) + s3_storage = S3BaseStorage(self._config) return s3_storage raise NotImplementedError("Unsupported storage provider") diff --git a/medusa/storage/local_storage.py b/medusa/storage/local_storage.py index a15481bb5..ea7570842 100644 --- a/medusa/storage/local_storage.py +++ b/medusa/storage/local_storage.py @@ -31,13 +31,14 @@ class LocalStorage(AbstractStorage): def __init__(self, config): + # in Python we usually put this last, bur we need it to set the bucket_name + super().__init__(config) + self.config = config self.root_dir = Path(config.base_path) / self.bucket_name self.root_dir.mkdir(parents=True, exist_ok=True) - super().__init__(config) - def connect(self): pass