From 3fd53d94a566f46bc84bcbbd01e7160441a01769 Mon Sep 17 00:00:00 2001 From: Radovan Date: Wed, 22 Nov 2023 13:24:29 +0200 Subject: [PATCH] [Azure Storage] Honor host/port from config (#685) --- docs/azure_blobs_setup.md | 15 ++--- medusa/storage/azure_storage.py | 12 +++- tests/storage/abstract_storage_test.py | 8 ++- tests/storage/azure_storage_test.py | 79 ++++++++++++++++++++++++++ tests/storage/google_storage_test.py | 2 +- tests/storage/s3_storage_test.py | 7 +-- 6 files changed, 105 insertions(+), 18 deletions(-) create mode 100644 tests/storage/azure_storage_test.py diff --git a/docs/azure_blobs_setup.md b/docs/azure_blobs_setup.md index 135b7c531..447dbc7f7 100644 --- a/docs/azure_blobs_setup.md +++ b/docs/azure_blobs_setup.md @@ -11,15 +11,7 @@ Create a new storage account or use an existing one which will be used to store "key": "YOUR_KEY" } ``` -If you need to set a different host for Azure (for example the host for Azure Gov is `.blob.core.usgovcloudapi.net`), please ADDITIONALLY set these two fields in the JSON file (the connection string can be found with the key): - -``` -"host": "YOUR_HOST" -"connection_string": "YOUR_CONNECTION_STRING" - -``` - -Place this file on all Apache Cassandra™ nodes running medusa under `/etc/medusa/`and set the rigths appropriately so that onyl users running Medusa can read/modify it. +Place this file on all Apache Cassandra™ nodes running medusa under `/etc/medusa/`and set the rights appropriately so that only users running Medusa can read/modify it. ### Create a container @@ -36,3 +28,8 @@ key_file = /etc/medusa/medusa-azure-credentials Medusa should now be able to access the bucket and perform all required operations. +If you need to set a different host for Azure (for example the host for Azure Gov is `.blob.core.usgovcloudapi.net`), please use the `host` parameter in the `[storage]` section of `/etc/medusa/medusa.ini`: + +``` +"host": "usgovcloudapi.net" +``` diff --git a/medusa/storage/azure_storage.py b/medusa/storage/azure_storage.py index 9faee5a83..9d25b792b 100644 --- a/medusa/storage/azure_storage.py +++ b/medusa/storage/azure_storage.py @@ -49,15 +49,25 @@ def __init__(self, config): self.account_name = self.credentials.named_key.name self.bucket_name = config.bucket_name + self.azure_blob_service_url = self._make_blob_service_url(self.account_name, config) + # disable chatty loggers logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.WARNING) logging.getLogger('chardet.universaldetector').setLevel(logging.WARNING) super().__init__(config) + def _make_blob_service_url(self, account_name, config): + domain = 'windows.net' if config.host is None else config.host + if config.port is None: + url = f"https://{account_name}.blob.core.{domain}/" + else: + url = f"https://{account_name}.blob.core.{domain}:{config.port}/" + return url + def connect(self): self.azure_blob_service = BlobServiceClient( - account_url=f"https://{self.account_name}.blob.core.windows.net/", + account_url=self.azure_blob_service_url, credential=self.credentials ) self.azure_container_client = self.azure_blob_service.get_container_client(self.bucket_name) diff --git a/tests/storage/abstract_storage_test.py b/tests/storage/abstract_storage_test.py index 6051142c6..56624830d 100644 --- a/tests/storage/abstract_storage_test.py +++ b/tests/storage/abstract_storage_test.py @@ -18,7 +18,13 @@ from medusa.storage.abstract_storage import AbstractStorage -class S3StorageTest(unittest.TestCase): +class AttributeDict(dict): + __slots__ = () + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + + +class AbstractStorageTest(unittest.TestCase): def test_convert_human_friendly_size_to_bytes(self): self.assertEqual(50, AbstractStorage._human_size_to_bytes('50B')) diff --git a/tests/storage/azure_storage_test.py b/tests/storage/azure_storage_test.py new file mode 100644 index 000000000..7809eac62 --- /dev/null +++ b/tests/storage/azure_storage_test.py @@ -0,0 +1,79 @@ +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import tempfile +import unittest + +from medusa.storage.azure_storage import AzureStorage +from tests.storage.abstract_storage_test import AttributeDict + + +class AzureStorageTest(unittest.TestCase): + + credentials_file_content = """ + { + "storage_account": "medusa-unit-test", + "key": "randomString==" + } + """ + + def test_make_connection_url(self): + with tempfile.NamedTemporaryFile() as credentials_file: + credentials_file.write(self.credentials_file_content.encode()) + credentials_file.flush() + config = AttributeDict({ + 'region': 'region-from-config', + 'storage_provider': 'azure_blobs', + 'key_file': credentials_file.name, + 'bucket_name': 'bucket-from-config', + 'concurrent_transfers': '1', + 'host': None, + 'port': None, + }) + azure_storage = AzureStorage(config) + self.assertEqual( + 'https://medusa-unit-test.blob.core.windows.net/', + azure_storage.azure_blob_service_url + ) + + def test_make_connection_url_with_custom_host(self): + with tempfile.NamedTemporaryFile() as credentials_file: + credentials_file.write(self.credentials_file_content.encode()) + credentials_file.flush() + config = AttributeDict({ + 'region': 'region-from-config', + 'storage_provider': 'azure_blobs', + 'key_file': credentials_file.name, + 'bucket_name': 'bucket-from-config', + 'concurrent_transfers': '1', + 'host': 'custom.host.net', + 'port': None, + }) + azure_storage = AzureStorage(config) + self.assertEqual( + 'https://medusa-unit-test.blob.core.custom.host.net/', + azure_storage.azure_blob_service_url + ) + + def test_make_connection_url_with_custom_host_port(self): + with tempfile.NamedTemporaryFile() as credentials_file: + credentials_file.write(self.credentials_file_content.encode()) + credentials_file.flush() + config = AttributeDict({ + 'region': 'region-from-config', + 'storage_provider': 'azure_blobs', + 'key_file': credentials_file.name, + 'bucket_name': 'bucket-from-config', + 'concurrent_transfers': '1', + 'host': 'custom.host.net', + 'port': 123, + }) + azure_storage = AzureStorage(config) + self.assertEqual( + 'https://medusa-unit-test.blob.core.custom.host.net:123/', + azure_storage.azure_blob_service_url + ) diff --git a/tests/storage/google_storage_test.py b/tests/storage/google_storage_test.py index 53bfd6d3b..891b49f34 100644 --- a/tests/storage/google_storage_test.py +++ b/tests/storage/google_storage_test.py @@ -22,7 +22,7 @@ from medusa.storage.google_storage import _group_by_parent, _is_in_folder -class RestoreNodeTest(unittest.TestCase): +class GoogleStorageTest(unittest.TestCase): def test_is_in_folder(self): folder = Path('foo/bar') diff --git a/tests/storage/s3_storage_test.py b/tests/storage/s3_storage_test.py index 08e98c6a4..11a5daaa3 100644 --- a/tests/storage/s3_storage_test.py +++ b/tests/storage/s3_storage_test.py @@ -21,12 +21,7 @@ from unittest.mock import patch, MagicMock from medusa.storage.s3_base_storage import S3BaseStorage - - -class AttributeDict(dict): - __slots__ = () - __getattr__ = dict.__getitem__ - __setattr__ = dict.__setitem__ +from tests.storage.abstract_storage_test import AttributeDict class S3StorageTest(unittest.TestCase):