Skip to content

Commit

Permalink
TOOLS-2783 TOOLS-2782 generate config bugs (#244)
Browse files Browse the repository at this point in the history 
* fix: TOOLS-2783 ship-bin, ship-set, ignore-bin, ignore-set not reported in the config

* fix: TOOLS-2782 `generate config` does not save shadow devices
  • Loading branch information
Jesse S authored Dec 8, 2023
1 parent 60a35d6 commit 9ff9ae1
Show file tree
Hide file tree
Showing 5 changed files with 156 additions and 44 deletions.
4 changes: 2 additions & 2 deletions lib/live_cluster/show_controller.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,13 +524,13 @@ async def do_namespace(self, line):
util.callable(
self.view.show_config,
"%s Namespace Configuration" % (ns),
configs,
ns_configs[ns],
self.cluster,
title_every_nth=title_every_nth,
flip_output=flip_output,
**self.mods,
)
for ns, configs in list(ns_configs.items())
for ns in sorted(ns_configs.keys())
]

# pre 5.0 but will still work
Expand Down
104 changes: 85 additions & 19 deletions lib/utils/conf_gen.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
import collections
import copy
import logging
from typing import Any
Expand All @@ -25,7 +26,8 @@


class IntermediateKey:
pass
def __repr__(self) -> str:
return self.__str__()


class InterNamedSectionKey(IntermediateKey):
Expand All @@ -45,9 +47,6 @@ def __eq__(self, __value: object) -> bool:
def __str__(self) -> str:
return f"({self.__class__.__name__}, {self.type}, {self.name})"

def __repr__(self) -> str:
return self.__str__()


class InterUnnamedSectionKey(IntermediateKey):
def __init__(
Expand All @@ -68,9 +67,6 @@ def __eq__(self, __value: object) -> bool:
def __str__(self) -> str:
return f"({self.__class__.__name__}, {self.type})"

def __repr__(self) -> str:
return self.__str__()


class InterLoggingContextKey(str):
pass
Expand All @@ -92,9 +88,6 @@ def __eq__(self, __value: object) -> bool:
def __str__(self) -> str:
return f"({self.__class__.__name__}, {self.name})"

def __repr__(self) -> str:
return self.__str__()


class ConfigPipelineStep:
async def __call__(self, intermediate_dict: dict[IntermediateKey | str, Any]):
Expand Down Expand Up @@ -265,12 +258,21 @@ async def __call__(self, context_dict: dict[str, Any]):
host_xdr_namespace_config = xdr_namespace_config[host]
for dc in host_xdr_namespace_config:
for ns in host_xdr_namespace_config[dc]:
context_dict[INTERMEDIATE][host][InterUnnamedSectionKey("xdr")][
InterNamedSectionKey("dc", dc)
][InterNamedSectionKey("namespace", ns)] = copy.deepcopy(
ns_config = context_dict[INTERMEDIATE][host][
InterUnnamedSectionKey("xdr")
][InterNamedSectionKey("dc", dc)][
InterNamedSectionKey("namespace", ns)
] = copy.deepcopy(
host_xdr_namespace_config[dc][ns]
)

for key, val in host_xdr_namespace_config[dc][ns].items():
newKey = convert_response_to_config_key(key)

if newKey != key:
ns_config[newKey] = val
del ns_config[key]


class CopyLoggingConfig(ConfigPipelineStep):
def _copy_subcontext(self, config_dict: dict[str, Any]):
Expand Down Expand Up @@ -357,6 +359,25 @@ def __init__(self):
)


class AppendShadowDevice(ConfigPipelineStep):
async def __call__(self, context_dict: dict[str, Any]):
intermediate_dict = context_dict[INTERMEDIATE]
namespace_config = context_dict["namespaces"]

for host in namespace_config:
host_namespaces_config = namespace_config[host]
for ns in host_namespaces_config.keys():
for config, value in host_namespaces_config[ns].items():
if config.endswith(".shadow"):
device_config = config.replace(".shadow", "")
intermediate_dict[host][InterNamedSectionKey("namespace", ns)][
device_config
] += (" " + value)
del intermediate_dict[host][
InterNamedSectionKey("namespace", ns)
][config]


class SplitSubcontexts(ConfigPipelineStep):
"""Takes a config dict and splits any subcontexts that are joined with a dot
into their own subdicts. E.g. "heartbeat.interval" -> {"heartbeat": {"interval": ...}}
Expand Down Expand Up @@ -465,7 +486,7 @@ async def __call__(self, context_dict: dict[str, Any]):

class ConvertIndexedToList(ConfigPipelineStep):
def _helper(self, config_dict: dict[str | IntermediateKey, Any]):
tmp_list_dict: dict[str, list[tuple[int, str]]] = {}
tmp_list_dict: dict[str, list[tuple[int, str]]] = collections.defaultdict(list)

for config in list(config_dict.keys()):
value = config_dict[config]
Expand Down Expand Up @@ -530,7 +551,10 @@ def _helper(

# Handles ldap scenarios with configs that allow commas in the config and
# are returned with commas. e.g. "query-base-dn" and "user-dn-pattern"
if len(split_value) > 1 and not ("-dn" in config and "ldap" in context):
if len(split_value) > 1 and not (
("-dn" in config and "ldap" in context)
or config in {"dcs", "namespaces"}
):
config_dict[InterListKey(config)] = split_value

del config_dict[config]
Expand Down Expand Up @@ -941,15 +965,16 @@ async def _generate_intermediate(
ServerVersionCheck(),
CopyToIntermediateDict(),
OverrideNamespaceRackID(),
AppendShadowDevice(),
SplitSubcontexts(),
ConvertIndexedSubcontextsToNamedSection(),
RemoveNullOrEmptyOrUndefinedValues(),
ConvertIndexedToList(),
SplitColonSeparatedValues(),
ConvertCommaSeparatedToList(),
RemoveSecurityIfNotEnabled(),
RemoveDefaultAndNonExistentKeys(JsonDynamicConfigHandler),
RemoveInvalidKeysFoundInSchemas(),
ConvertCommaSeparatedToList(),
RemoveEmptyContexts(), # Should be after RemoveDefaultValues
RemoveConfigsConditionally(),
],
Expand All @@ -959,21 +984,49 @@ async def _generate_intermediate(
await pipeline(context_dict)
return context_dict[INTERMEDIATE]

def _sort_keys(self, intermediate_dict: dict[IntermediateKey | str, Any]):
str_keys = []
list_keys = []
unnamed_keys = []
named_keys = []

for key in intermediate_dict:
if isinstance(key, str):
str_keys.append(key)
elif isinstance(key, InterListKey):
list_keys.append(key)
elif isinstance(key, InterUnnamedSectionKey):
unnamed_keys.append(key)
elif isinstance(key, InterNamedSectionKey):
named_keys.append(key)

str_keys.sort()
list_keys.sort(key=lambda x: x.name)
named_keys.sort(key=lambda x: x.type + x.name)

return str_keys + list_keys + unnamed_keys + named_keys

def _generate_helper(
self,
result: list[str],
intermediate_dict: dict[IntermediateKey | str, Any],
indent=0,
):
adjusted_indent = indent * 2
keys = self._sort_keys(intermediate_dict)

for key, val in intermediate_dict.items():
for i, key in enumerate(keys):
val = intermediate_dict[key]
if isinstance(key, InterUnnamedSectionKey):
result.append(f"\n{' ' * adjusted_indent}{key.type} {{")
if i != 0:
result.append("")
result.append(f"{' ' * adjusted_indent}{key.type} {{")
self._generate_helper(result, val, indent + 1)
result.append(f"{' ' * adjusted_indent}}}")
elif isinstance(key, InterNamedSectionKey):
result.append(f"\n{' ' * adjusted_indent}{key.type} {key.name} {{")
if i != 0:
result.append("")
result.append(f"{' ' * adjusted_indent}{key.type} {key.name} {{")
self._generate_helper(result, val, indent + 1)
result.append(f"{' ' * adjusted_indent}}}")
elif isinstance(key, InterListKey):
Expand All @@ -997,3 +1050,16 @@ async def generate(

self._generate_helper(lines, list(intermediate_dict.values())[0])
return "\n".join(lines)


# Helpers
server_to_config_key_map = {
"shipped-bins": "ship-bin",
"shipped-sets": "ship-set",
"ignored-bins": "ignore-bin",
"ignored-sets": "ignore-set",
}


def convert_response_to_config_key(key: str) -> str:
return server_to_config_key_map.get(key, key)
59 changes: 47 additions & 12 deletions test/e2e/live_cluster/test_confgen.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
${security_stanza}
service {
cluster-name asadm-test
feature-key-file ${feature_path}
run-as-daemon false
pidfile ${state_directory}/asd.pid
Expand Down Expand Up @@ -58,23 +59,50 @@
}
}
namespace bar {
replication-factor 2
default-ttl 0
storage-engine memory {
data-size 1G
}
nsup-period 60
}
namespace ${namespace} {
replication-factor 2
memory-size 512M
default-ttl 0
storage-engine device {
file /opt/aerospike/data/test.dat
file /opt/aerospike/data/test.dat /opt/aerospike/data/test-shadow.dat
filesize 1G
data-in-memory false # Store data in memory in addition to file.
}
nsup-period 60
}
xdr {
dc DC1 {
namespace ${namespace} {
}
dc DC1 {
namespace ${namespace} {
bin-policy changed-or-specified
ignore-set testset
ignore-set barset
ship-bin bar
ship-bin foo
}
namespace bar {
}
}
dc DC2 {
namespace ${namespace} {
bin-policy changed-or-specified
ignore-set testset
ignore-set barset
ship-bin bar
ship-bin foo
}
namespace bar {
}
}
}
"""

Expand All @@ -97,18 +125,23 @@ class TestConfGen(asynctest.TestCase):
"""

@classmethod
def rm_timestamp_from_output(cls, output):
def clean_output(cls, output):
lines = output.split("\n")
for i, l in enumerate(lines):
l = re.sub(r"([0-9]{2}:){2}[0-9]{2}", "", l)

if ".stripe" in l:
l = ""

lines[i] = l

return "\n".join(lines)

def tearDown(self):
lib.stop()

async def test_genconf(self):
lib.start(num_nodes=1)
lib.start(num_nodes=1, template_content=aerospike_conf)
time.sleep(1)
conf_gen_cmd = f"generate config with 127.0.0.1:{lib.PORT}"
show_config_cmd = "show config; show config security; show config xdr"
Expand Down Expand Up @@ -161,9 +194,11 @@ async def test_genconf(self):
second_show_config = cp.stdout

self.assertEqual(first_conf, second_conf)
first_show_config = TestConfGen.clean_output(first_show_config)
second_show_config = TestConfGen.clean_output(second_show_config)
self.assertEqual(
TestConfGen.rm_timestamp_from_output(first_show_config),
TestConfGen.rm_timestamp_from_output(second_show_config),
first_show_config,
second_show_config,
)

async def test_genconf_save_to_file(self):
Expand All @@ -175,8 +210,8 @@ async def test_genconf_save_to_file(self):
)

if cp.returncode != 0:
print(cp.stdout)
print(cp.stderr)
# print(cp.stdout)
# print(cp.stderr)
self.fail()

first_conf = cp.stdout
Expand Down
8 changes: 4 additions & 4 deletions test/unit/live_cluster/test_show_controller.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,16 @@ async def test_do_namespace_default(self):
self.view_mock.show_config.assert_has_calls(
[
call(
"foo Namespace Configuration",
"foo-configs",
"bar Namespace Configuration",
"bar-configs",
self.cluster_mock,
title_every_nth=0,
flip_output=False,
**self.controller.mods,
),
call(
"bar Namespace Configuration",
"bar-configs",
"foo Namespace Configuration",
"foo-configs",
self.cluster_mock,
title_every_nth=0,
flip_output=False,
Expand Down
Loading

0 comments on commit 9ff9ae1

Please sign in to comment.