diff --git a/lib/live_cluster/show_controller.py b/lib/live_cluster/show_controller.py index c65f5742..56b0b123 100644 --- a/lib/live_cluster/show_controller.py +++ b/lib/live_cluster/show_controller.py @@ -524,13 +524,13 @@ async def do_namespace(self, line): util.callable( self.view.show_config, "%s Namespace Configuration" % (ns), - configs, + ns_configs[ns], self.cluster, title_every_nth=title_every_nth, flip_output=flip_output, **self.mods, ) - for ns, configs in list(ns_configs.items()) + for ns in sorted(ns_configs.keys()) ] # pre 5.0 but will still work diff --git a/lib/utils/conf_gen.py b/lib/utils/conf_gen.py index b7cc46b4..f2cae929 100644 --- a/lib/utils/conf_gen.py +++ b/lib/utils/conf_gen.py @@ -1,4 +1,5 @@ import asyncio +import collections import copy import logging from typing import Any @@ -25,7 +26,8 @@ class IntermediateKey: - pass + def __repr__(self) -> str: + return self.__str__() class InterNamedSectionKey(IntermediateKey): @@ -45,9 +47,6 @@ def __eq__(self, __value: object) -> bool: def __str__(self) -> str: return f"({self.__class__.__name__}, {self.type}, {self.name})" - def __repr__(self) -> str: - return self.__str__() - class InterUnnamedSectionKey(IntermediateKey): def __init__( @@ -68,9 +67,6 @@ def __eq__(self, __value: object) -> bool: def __str__(self) -> str: return f"({self.__class__.__name__}, {self.type})" - def __repr__(self) -> str: - return self.__str__() - class InterLoggingContextKey(str): pass @@ -92,9 +88,6 @@ def __eq__(self, __value: object) -> bool: def __str__(self) -> str: return f"({self.__class__.__name__}, {self.name})" - def __repr__(self) -> str: - return self.__str__() - class ConfigPipelineStep: async def __call__(self, intermediate_dict: dict[IntermediateKey | str, Any]): @@ -265,12 +258,21 @@ async def __call__(self, context_dict: dict[str, Any]): host_xdr_namespace_config = xdr_namespace_config[host] for dc in host_xdr_namespace_config: for ns in host_xdr_namespace_config[dc]: - context_dict[INTERMEDIATE][host][InterUnnamedSectionKey("xdr")][ - InterNamedSectionKey("dc", dc) - ][InterNamedSectionKey("namespace", ns)] = copy.deepcopy( + ns_config = context_dict[INTERMEDIATE][host][ + InterUnnamedSectionKey("xdr") + ][InterNamedSectionKey("dc", dc)][ + InterNamedSectionKey("namespace", ns) + ] = copy.deepcopy( host_xdr_namespace_config[dc][ns] ) + for key, val in host_xdr_namespace_config[dc][ns].items(): + newKey = convert_response_to_config_key(key) + + if newKey != key: + ns_config[newKey] = val + del ns_config[key] + class CopyLoggingConfig(ConfigPipelineStep): def _copy_subcontext(self, config_dict: dict[str, Any]): @@ -357,6 +359,25 @@ def __init__(self): ) +class AppendShadowDevice(ConfigPipelineStep): + async def __call__(self, context_dict: dict[str, Any]): + intermediate_dict = context_dict[INTERMEDIATE] + namespace_config = context_dict["namespaces"] + + for host in namespace_config: + host_namespaces_config = namespace_config[host] + for ns in host_namespaces_config.keys(): + for config, value in host_namespaces_config[ns].items(): + if config.endswith(".shadow"): + device_config = config.replace(".shadow", "") + intermediate_dict[host][InterNamedSectionKey("namespace", ns)][ + device_config + ] += (" " + value) + del intermediate_dict[host][ + InterNamedSectionKey("namespace", ns) + ][config] + + class SplitSubcontexts(ConfigPipelineStep): """Takes a config dict and splits any subcontexts that are joined with a dot into their own subdicts. E.g. "heartbeat.interval" -> {"heartbeat": {"interval": ...}} @@ -465,7 +486,7 @@ async def __call__(self, context_dict: dict[str, Any]): class ConvertIndexedToList(ConfigPipelineStep): def _helper(self, config_dict: dict[str | IntermediateKey, Any]): - tmp_list_dict: dict[str, list[tuple[int, str]]] = {} + tmp_list_dict: dict[str, list[tuple[int, str]]] = collections.defaultdict(list) for config in list(config_dict.keys()): value = config_dict[config] @@ -530,7 +551,10 @@ def _helper( # Handles ldap scenarios with configs that allow commas in the config and # are returned with commas. e.g. "query-base-dn" and "user-dn-pattern" - if len(split_value) > 1 and not ("-dn" in config and "ldap" in context): + if len(split_value) > 1 and not ( + ("-dn" in config and "ldap" in context) + or config in {"dcs", "namespaces"} + ): config_dict[InterListKey(config)] = split_value del config_dict[config] @@ -941,15 +965,16 @@ async def _generate_intermediate( ServerVersionCheck(), CopyToIntermediateDict(), OverrideNamespaceRackID(), + AppendShadowDevice(), SplitSubcontexts(), ConvertIndexedSubcontextsToNamedSection(), RemoveNullOrEmptyOrUndefinedValues(), ConvertIndexedToList(), SplitColonSeparatedValues(), + ConvertCommaSeparatedToList(), RemoveSecurityIfNotEnabled(), RemoveDefaultAndNonExistentKeys(JsonDynamicConfigHandler), RemoveInvalidKeysFoundInSchemas(), - ConvertCommaSeparatedToList(), RemoveEmptyContexts(), # Should be after RemoveDefaultValues RemoveConfigsConditionally(), ], @@ -959,6 +984,28 @@ async def _generate_intermediate( await pipeline(context_dict) return context_dict[INTERMEDIATE] + def _sort_keys(self, intermediate_dict: dict[IntermediateKey | str, Any]): + str_keys = [] + list_keys = [] + unnamed_keys = [] + named_keys = [] + + for key in intermediate_dict: + if isinstance(key, str): + str_keys.append(key) + elif isinstance(key, InterListKey): + list_keys.append(key) + elif isinstance(key, InterUnnamedSectionKey): + unnamed_keys.append(key) + elif isinstance(key, InterNamedSectionKey): + named_keys.append(key) + + str_keys.sort() + list_keys.sort(key=lambda x: x.name) + named_keys.sort(key=lambda x: x.type + x.name) + + return str_keys + list_keys + unnamed_keys + named_keys + def _generate_helper( self, result: list[str], @@ -966,14 +1013,20 @@ def _generate_helper( indent=0, ): adjusted_indent = indent * 2 + keys = self._sort_keys(intermediate_dict) - for key, val in intermediate_dict.items(): + for i, key in enumerate(keys): + val = intermediate_dict[key] if isinstance(key, InterUnnamedSectionKey): - result.append(f"\n{' ' * adjusted_indent}{key.type} {{") + if i != 0: + result.append("") + result.append(f"{' ' * adjusted_indent}{key.type} {{") self._generate_helper(result, val, indent + 1) result.append(f"{' ' * adjusted_indent}}}") elif isinstance(key, InterNamedSectionKey): - result.append(f"\n{' ' * adjusted_indent}{key.type} {key.name} {{") + if i != 0: + result.append("") + result.append(f"{' ' * adjusted_indent}{key.type} {key.name} {{") self._generate_helper(result, val, indent + 1) result.append(f"{' ' * adjusted_indent}}}") elif isinstance(key, InterListKey): @@ -997,3 +1050,16 @@ async def generate( self._generate_helper(lines, list(intermediate_dict.values())[0]) return "\n".join(lines) + + +# Helpers +server_to_config_key_map = { + "shipped-bins": "ship-bin", + "shipped-sets": "ship-set", + "ignored-bins": "ignore-bin", + "ignored-sets": "ignore-set", +} + + +def convert_response_to_config_key(key: str) -> str: + return server_to_config_key_map.get(key, key) diff --git a/test/e2e/live_cluster/test_confgen.py b/test/e2e/live_cluster/test_confgen.py index 5c603cf6..47e973aa 100644 --- a/test/e2e/live_cluster/test_confgen.py +++ b/test/e2e/live_cluster/test_confgen.py @@ -8,6 +8,7 @@ ${security_stanza} service { + cluster-name asadm-test feature-key-file ${feature_path} run-as-daemon false pidfile ${state_directory}/asd.pid @@ -58,23 +59,50 @@ } } +namespace bar { + replication-factor 2 + default-ttl 0 + storage-engine memory { + data-size 1G + } + nsup-period 60 +} + namespace ${namespace} { replication-factor 2 - memory-size 512M default-ttl 0 storage-engine device { - file /opt/aerospike/data/test.dat + file /opt/aerospike/data/test.dat /opt/aerospike/data/test-shadow.dat filesize 1G - data-in-memory false # Store data in memory in addition to file. } nsup-period 60 } xdr { - dc DC1 { - namespace ${namespace} { - } + dc DC1 { + namespace ${namespace} { + bin-policy changed-or-specified + ignore-set testset + ignore-set barset + ship-bin bar + ship-bin foo + } + + namespace bar { + } + } + dc DC2 { + namespace ${namespace} { + bin-policy changed-or-specified + ignore-set testset + ignore-set barset + ship-bin bar + ship-bin foo } + + namespace bar { + } + } } """ @@ -97,18 +125,23 @@ class TestConfGen(asynctest.TestCase): """ @classmethod - def rm_timestamp_from_output(cls, output): + def clean_output(cls, output): lines = output.split("\n") for i, l in enumerate(lines): l = re.sub(r"([0-9]{2}:){2}[0-9]{2}", "", l) + + if ".stripe" in l: + l = "" + lines[i] = l + return "\n".join(lines) def tearDown(self): lib.stop() async def test_genconf(self): - lib.start(num_nodes=1) + lib.start(num_nodes=1, template_content=aerospike_conf) time.sleep(1) conf_gen_cmd = f"generate config with 127.0.0.1:{lib.PORT}" show_config_cmd = "show config; show config security; show config xdr" @@ -161,9 +194,11 @@ async def test_genconf(self): second_show_config = cp.stdout self.assertEqual(first_conf, second_conf) + first_show_config = TestConfGen.clean_output(first_show_config) + second_show_config = TestConfGen.clean_output(second_show_config) self.assertEqual( - TestConfGen.rm_timestamp_from_output(first_show_config), - TestConfGen.rm_timestamp_from_output(second_show_config), + first_show_config, + second_show_config, ) async def test_genconf_save_to_file(self): @@ -175,8 +210,8 @@ async def test_genconf_save_to_file(self): ) if cp.returncode != 0: - print(cp.stdout) - print(cp.stderr) + # print(cp.stdout) + # print(cp.stderr) self.fail() first_conf = cp.stdout diff --git a/test/unit/live_cluster/test_show_controller.py b/test/unit/live_cluster/test_show_controller.py index 6b6818bb..f4de44f3 100644 --- a/test/unit/live_cluster/test_show_controller.py +++ b/test/unit/live_cluster/test_show_controller.py @@ -133,16 +133,16 @@ async def test_do_namespace_default(self): self.view_mock.show_config.assert_has_calls( [ call( - "foo Namespace Configuration", - "foo-configs", + "bar Namespace Configuration", + "bar-configs", self.cluster_mock, title_every_nth=0, flip_output=False, **self.controller.mods, ), call( - "bar Namespace Configuration", - "bar-configs", + "foo Namespace Configuration", + "foo-configs", self.cluster_mock, title_every_nth=0, flip_output=False, diff --git a/test/unit/utils/test_conf_gen.py b/test/unit/utils/test_conf_gen.py index 0f41bff1..5c34b163 100644 --- a/test/unit/utils/test_conf_gen.py +++ b/test/unit/utils/test_conf_gen.py @@ -129,7 +129,12 @@ async def test_copy_to_intermediate(self): "sets": {"1.1.1.1": {("test", "testset"): 1, ("bar", "barset"): 1}}, "xdr": {"1.1.1.1": {"a": 1}}, "xdr-dcs": {"1.1.1.1": {"dc1": {"a": 1}, "dc2": {"b": 1}}}, - "xdr-namespaces": {"1.1.1.1": {"dc1": {"test": 1}, "dc2": {"bar": 2}}}, + "xdr-namespaces": { + "1.1.1.1": { + "dc1": {"test": {"shipped-bins": "a,b,c", "shipped-sets": "1,2,3"}}, + "dc2": {"bar": {"ignored-bins": "a,b,c", "ignored-sets": "1,2,3"}}, + } + }, } await CopyToIntermediateDict()(context_dict) @@ -158,11 +163,17 @@ async def test_copy_to_intermediate(self): "a": 1, InterNamedSectionKey("dc", "dc1"): { "a": 1, - InterNamedSectionKey("namespace", "test"): 1, + InterNamedSectionKey("namespace", "test"): { + "ship-bin": "a,b,c", + "ship-set": "1,2,3", + }, }, InterNamedSectionKey("dc", "dc2"): { "b": 1, - InterNamedSectionKey("namespace", "bar"): 2, + InterNamedSectionKey("namespace", "bar"): { + "ignore-bin": "a,b,c", + "ignore-set": "1,2,3", + }, }, }, } @@ -224,7 +235,7 @@ async def test_convert_indexed_to_list(self): "1.1.1.1": { InterUnnamedSectionKey("network"): { InterUnnamedSectionKey("tls[0]"): { - "name": "tls-name", + "name": "tls-foo", "b": "2", "c": "3", } @@ -240,7 +251,7 @@ async def test_convert_indexed_to_list(self): { "1.1.1.1": { InterUnnamedSectionKey("network"): { - InterNamedSectionKey("tls", "tls-name"): { + InterNamedSectionKey("tls", "tls-foo"): { "b": "2", "c": "3", } @@ -256,7 +267,7 @@ class ConvertIndexedToListTest(asynctest.TestCase): async def test_convert_indexed_to_list(self): context_dict = { "intermediate": { - "1.1.1.1": {"a": {"b[1]": "1", "b[0]": "2", "b[2]": "3"}}, + "1.1.1.1": {"a": {"b[1]": "1 4", "b[0]": "2", "b[2]": "3"}}, } } @@ -265,7 +276,7 @@ async def test_convert_indexed_to_list(self): self.assertDictEqual( context_dict["intermediate"], { - "1.1.1.1": {"a": {InterListKey("b"): ["2", "1", "3"]}}, + "1.1.1.1": {"a": {InterListKey("b"): ["2", "1 4", "3"]}}, }, )