From 785bf88b73129540630ca4b2d93b33c56ec4c578 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 3 May 2023 15:26:57 +0200 Subject: [PATCH 01/12] refactor: rename inventory function --- kapitan/refs/cmd_parser.py | 24 ++++++++++++------------ kapitan/resources.py | 33 ++++++++++++++++++++++----------- kapitan/targets.py | 10 +++++----- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/kapitan/refs/cmd_parser.py b/kapitan/refs/cmd_parser.py index f94496376..5cedeb751 100644 --- a/kapitan/refs/cmd_parser.py +++ b/kapitan/refs/cmd_parser.py @@ -16,7 +16,7 @@ from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints from kapitan.refs.secrets.vaultkv import VaultSecret from kapitan.refs.secrets.vaulttransit import VaultTransit -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.utils import fatal_error, search_target_token_paths logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -95,7 +95,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -123,7 +123,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -152,7 +152,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -196,7 +196,7 @@ def ref_write(args, ref_controller): vault_params = {} encoding = "original" if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -230,7 +230,7 @@ def ref_write(args, ref_controller): _data = data.encode() vault_params = {} if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -302,7 +302,7 @@ def secret_update(args, ref_controller): for name in args.recipients ] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -330,7 +330,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("gkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -356,7 +356,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("azkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -382,7 +382,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("awskms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -439,7 +439,7 @@ def secret_update_validate(args, ref_controller): "Validate and/or update target secrets" # update gpg recipients/gkms/awskms key for all secrets in secrets_path # use --refs-path to set scanning path - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) targets = set(inv["nodes"].keys()) secrets_path = os.path.abspath(args.refs_path) target_token_paths = search_target_token_paths(secrets_path, targets) diff --git a/kapitan/resources.py b/kapitan/resources.py index e77f4c885..c5e29ee5d 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -280,14 +280,14 @@ def inventory(search_paths, target, inventory_path=None): raise InventoryError(f"Inventory not found in search paths: {search_paths}") if target is None: - return inventory_reclass(full_inv_path)["nodes"] + return get_inventory(full_inv_path)["nodes"] - return inventory_reclass(full_inv_path)["nodes"][target] + return get_inventory(full_inv_path)["nodes"][target] def generate_inventory(args): try: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) if args.target_name != "": inv = inv["nodes"][args.target_name] if args.pattern != "": @@ -304,6 +304,23 @@ def generate_inventory(args): sys.exit(1) +def get_inventory(inventory_path, ignore_class_notfound=False): + """ + generic inventory function that makes inventory backend pluggable + default backend is reclass + """ + + # if inventory is already cached theres nothing to do + if cached.inv: + return cached.inv + + logger.debug("Using reclass as inventory backend") + inv = inventory_reclass(inventory_path, ignore_class_notfound) + + cached.inv = inv + return inv + + def inventory_reclass(inventory_path, ignore_class_notfound=False): """ Runs a reclass inventory in inventory_path @@ -314,12 +331,8 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): Does not throw errors if a class is not found while --fetch flag is enabled """ - # if inventory is already cached theres nothing to do - if cached.inv: - return cached.inv - # set default values initially - reclass_config = reclass_config_defaults = { + reclass_config = { "storage_type": "yaml_fs", "inventory_base_uri": inventory_path, "nodes_uri": "targets", @@ -358,12 +371,10 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - cached.inv = _reclass.inventory() + return _reclass.inventory() except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") else: logger.error("Inventory reclass error: %s", e.message) raise InventoryError(e.message) - - return cached.inv diff --git a/kapitan/targets.py b/kapitan/targets.py index 7327fb7d8..473fd965c 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -32,7 +32,7 @@ from kapitan.inputs.kadet import Kadet from kapitan.inputs.remove import Remove from kapitan.remoteinventory.fetch import fetch_inventories, list_sources -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache from kapitan.validator.kubernetes_validator import KubernetesManifestValidator @@ -241,7 +241,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): ... } """ - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) cached.inv_cache = {} cached.inv_cache["inventory"] = {} cached.inv_cache["folder"] = {} @@ -295,7 +295,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): def changed_targets(inventory_path, output_path): """returns a list of targets that have changed since last compilation""" targets = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) saved_inv_cache = None saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache") @@ -388,7 +388,7 @@ def save_inv_cache(compile_path, targets): def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): """returns a list of target objects from the inventory""" target_objs = [] - inv = inventory_reclass(inventory_path, ignore_class_notfound) + inv = get_inventory(inventory_path, ignore_class_notfound) # if '-t' is set on compile, only loop through selected targets if targets: @@ -431,7 +431,7 @@ def search_targets(inventory_path, targets, labels): ) targets_found = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) for target_name in inv["nodes"]: matched_all_labels = False From 840d58d2530928b2399412db6db92b9b08643723 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Thu, 2 Nov 2023 12:38:13 +0100 Subject: [PATCH 02/12] feat: introduce inventory interface --- kapitan/inventory/__init__.py | 1 + kapitan/inventory/base.py | 61 ++++++++++++++++++++++++++ kapitan/inventory/reclass.py | 82 +++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 kapitan/inventory/__init__.py create mode 100644 kapitan/inventory/base.py create mode 100644 kapitan/inventory/reclass.py diff --git a/kapitan/inventory/__init__.py b/kapitan/inventory/__init__.py new file mode 100644 index 000000000..562ee3ef4 --- /dev/null +++ b/kapitan/inventory/__init__.py @@ -0,0 +1 @@ +from .base import Inventory diff --git a/kapitan/inventory/base.py b/kapitan/inventory/base.py new file mode 100644 index 000000000..ebfe2f26f --- /dev/null +++ b/kapitan/inventory/base.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The Kapitan Authors +# SPDX-FileCopyrightText: 2023 The Kapitan Authors +# +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod + +from kapitan.errors import InventoryError + + +class Inventory(ABC): + _instance: object + + def __new__(cls, *args, **kwargs): + """ + create only one specific inventory instance (singleton) + """ + if not cls._instance: + # initialize new instance + cls._instance = super(Inventory, cls).__new__(cls) + cls._instance.__init__(**args, **kwargs) + return cls._instance + + def __init__(self, inventory_path: str, compose_node_name: dict = False): + self.inventory_path = inventory_path + self.compose_node_name = compose_node_name + self._parameters = None + + @property + def inventory(self) -> dict: + if not self._parameters: + self._parameters = self.get_parameters() + return self._parameters + + @abstractmethod + def get_parameters(self, ignore_class_notfound=False) -> dict: + """ + create the inventory depending which backend gets used + """ + raise NotImplementedError + + def search_targets(self) -> list: + """ + look for targets at '/targets/' + """ + targets = [] + return targets + + def get_target(self, target_name: str) -> dict: + """ + get parameters for a specific target + """ + target = self.inventory.get(target_name) + if not target: + raise InventoryError(f"target '{target_name}' not found") + return target + + def fetch_dependencies(self): + raise NotImplementedError diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py new file mode 100644 index 000000000..ccb6f3a92 --- /dev/null +++ b/kapitan/inventory/reclass.py @@ -0,0 +1,82 @@ +from kapitan.inventory.base import Inventory +import os +import yaml +import logging + +import reclass +import reclass.core +from reclass.errors import NotFoundError, ReclassException + +from kapitan.errors import InventoryError + +logger = logging.getLogger(__name__) + +class ReclassInventory(Inventory): + + def get_parameters(self, ignore_class_notfound=False) -> dict: + """ + Runs a reclass inventory in inventory_path + (same output as running ./reclass.py -b inv_base_uri/ --inventory) + Will attempt to read reclass config from 'reclass-config.yml' otherwise + it will failback to the default config. + Returns a reclass style dictionary + + Does not throw errors if a class is not found while --fetch flag is enabled + """ + reclass_config = self.get_config() + reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound) + + try: + storage = reclass.get_storage( + reclass_config["storage_type"], + reclass_config["nodes_uri"], + reclass_config["classes_uri"], + reclass_config["compose_node_name"], + ) + class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) + _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) + + return _reclass.inventory() + except ReclassException as e: + if isinstance(e, NotFoundError): + logger.error("Inventory reclass error: inventory not found") + else: + logger.error(f"Inventory reclass error: {e.message}") + raise InventoryError(e.message) + + + def get_config(self) -> dict: + # set default values initially + reclass_config = { + "storage_type": "yaml_fs", + "inventory_base_uri": self.inventory_path, + "nodes_uri": "targets", + "classes_uri": "classes", + "compose_node_name": False, + "allow_none_override": True, + } + try: + from yaml import CSafeLoader as YamlLoader + except ImportError: + from yaml import SafeLoader as YamlLoader + + # get reclass config from file 'inventory/reclass-config.yml' + cfg_file = os.path.join(self.inventory_path, "reclass-config.yml") + if os.path.isfile(cfg_file): + with open(cfg_file, "r") as fp: + config = yaml.load(fp.read(), Loader=YamlLoader) + logger.debug("Using reclass inventory config at: {}".format(cfg_file)) + if config: + # set attributes, take default values if not present + for key, value in config.items(): + reclass_config[key] = value + else: + logger.debug("{}: Empty config file. Using reclass inventory config defaults".format(cfg_file)) + else: + logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") + + # normalise relative nodes_uri and classes_uri paths + for uri in ("nodes_uri", "classes_uri"): + reclass_config[uri] = os.path.normpath(os.path.join(self.inventory_path, reclass_config[uri])) + + return reclass_config \ No newline at end of file From 60964c27cf12054cd6e60b14111bee53c352a588 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 22 Nov 2023 12:10:44 +0100 Subject: [PATCH 03/12] wip --- kapitan/cli.py | 4 +- kapitan/inputs/base.py | 15 ++ kapitan/inventory/__init__.py | 1 + kapitan/inventory/base.py | 159 +++++++++++-- kapitan/inventory/reclass.py | 20 +- kapitan/refs/cmd_parser.py | 24 +- kapitan/resources.py | 90 +------ kapitan/targets.py | 432 ++++------------------------------ tests/test_compile.py | 4 +- 9 files changed, 240 insertions(+), 509 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index d1d81ee8b..20e776cb8 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -25,7 +25,7 @@ from kapitan.refs.base import RefController, Revealer from kapitan.refs.cmd_parser import handle_refs_command from kapitan.resources import generate_inventory, resource_callbacks, search_imports -from kapitan.targets import compile_targets, schema_validate_compiled +from kapitan.targets import compile_targets from kapitan.utils import check_version, from_dot_kapitan, searchvar from kapitan.version import DESCRIPTION, PROJECT_NAME, VERSION @@ -584,7 +584,7 @@ def build_parser(): aliases=["v"], help="validates the compile output against schemas as specified in inventory", ) - validate_parser.set_defaults(func=schema_validate_compiled, name="validate") + # validate_parser.set_defaults(func=schema_validate_compiled, name="validate") validate_parser.add_argument( "--compiled-path", diff --git a/kapitan/inputs/base.py b/kapitan/inputs/base.py index 711eff26f..b8f8a95d1 100644 --- a/kapitan/inputs/base.py +++ b/kapitan/inputs/base.py @@ -18,10 +18,25 @@ from kapitan.errors import CompileError, KapitanError from kapitan.refs.base import Revealer from kapitan.utils import PrettyDumper +from enum import Enum logger = logging.getLogger(__name__) +class InputTypes(Enum): + """ + All currently supported input types in the inventory for parameters.kapitan.compile + """ + + KADET = "kadet" + JSONNET = "jsonnet" + JINJA2 = "jinja2" + HELM = "helm" + EXTERNAL = "external" + COPY = "copy" + REMOVE = "remove" + + class InputType(object): def __init__(self, type_name, compile_path, search_paths, ref_controller): self.type_name = type_name diff --git a/kapitan/inventory/__init__.py b/kapitan/inventory/__init__.py index 562ee3ef4..266ca4f5e 100644 --- a/kapitan/inventory/__init__.py +++ b/kapitan/inventory/__init__.py @@ -1 +1,2 @@ from .base import Inventory +from .reclass import ReclassInventory diff --git a/kapitan/inventory/base.py b/kapitan/inventory/base.py index ebfe2f26f..70003307d 100644 --- a/kapitan/inventory/base.py +++ b/kapitan/inventory/base.py @@ -6,56 +6,179 @@ # SPDX-License-Identifier: Apache-2.0 from abc import ABC, abstractmethod +from dataclasses import dataclass +import logging +import os +from kapitan import cached -from kapitan.errors import InventoryError +from kapitan.errors import InventoryError, KapitanError + +logger = logging.getLogger(__name__) + + +@dataclass +class InventoryTarget(dict): + targets_path: str + + def __init__(self, name: str, path: str): + self.name = name + self.path = path + + # compose node name + self.composed_name = os.path.splitext(path)[0].replace(path + os.sep, "").replace("/", ".") + + @property + def kapitan(self) -> dict(): + kapitan_spec = self.get("kapitan") + if not kapitan_spec: + raise InventoryValidationError("parameters.kapitan is empty") + + return kapitan_spec + + @property + def parameters(self) -> dict(): + parameters = self.get("parameters") + if not parameters: + raise InventoryValidationError("parameters is empty") + + return parameters class Inventory(ABC): - _instance: object + _instance = None def __new__(cls, *args, **kwargs): """ create only one specific inventory instance (singleton) """ - if not cls._instance: - # initialize new instance - cls._instance = super(Inventory, cls).__new__(cls) - cls._instance.__init__(**args, **kwargs) - return cls._instance + if Inventory._instance is None: + Inventory._instance = super().__new__(cls) + return Inventory._instance def __init__(self, inventory_path: str, compose_node_name: dict = False): self.inventory_path = inventory_path + self.targets_path = os.path.join(inventory_path, "targets") + self.classes_path = os.path.join(inventory_path, "classes") + + # config self.compose_node_name = compose_node_name - self._parameters = None + + # used as cache for inventory + self._targets = {} + self._parameters = {} + + @classmethod + def get(cls): + if cls._instance is None: + raise InventoryError("no type specified") + return cls._instance @property def inventory(self) -> dict: if not self._parameters: self._parameters = self.get_parameters() + cached.inv = self._parameters return self._parameters @abstractmethod - def get_parameters(self, ignore_class_notfound=False) -> dict: + def get_parameters(self, ignore_class_notfound: bool = False) -> dict: """ create the inventory depending which backend gets used """ raise NotImplementedError - def search_targets(self) -> list: + def search_targets(self) -> dict: """ - look for targets at '/targets/' + look for targets at '/targets/' and return targets """ - targets = [] - return targets + for root, _, files in os.walk(self.targets_path): + for file in files: + # split file extension and check if yml/yaml + path = os.path.join(root, file) + name, ext = os.path.splitext(file) + if ext not in (".yml", ".yaml"): + logger.debug(f"{file}: targets have to be .yml or .yaml files.") + continue + + # initialize target + target = InventoryTarget(name, path) + if self.compose_node_name: + target.name = target.composed_name + + # check for same name + if self._targets.get(target.name): + raise InventoryError( + f"Conflicting targets {target.name}: {target.path} and {self._targets[target.name].path}" + ) + + self._targets[target.name] = target + + return self._targets def get_target(self, target_name: str) -> dict: """ get parameters for a specific target """ - target = self.inventory.get(target_name) - if not target: - raise InventoryError(f"target '{target_name}' not found") - return target + return self.get_targets([target_name]) - def fetch_dependencies(self): + @abstractmethod + def get_targets(self, target_names: list[str]) -> dict: + """ + get parameters for multiple targets + """ raise NotImplementedError + + def fetch_dependencies(self, fetch, force_fetch): + # fetch inventory + if fetch: + # new_source checks for new sources in fetched inventory items + new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) + while new_sources: + fetch_inventories( + inventory_path, + target_objs, + dep_cache_dir, + force_fetch, + pool, + ) + cached.reset_inv() + target_objs = load_target_inventory(updated_targets) + cached.inv_sources.update(new_sources) + new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) + # reset inventory cache and load target objs to check for missing classes + cached.reset_inv() + target_objs = load_target_inventory(updated_targets) + # fetch dependencies + if fetch: + fetch_dependencies(output_path, target_objs, dep_cache_dir, force_fetch, pool) + # fetch targets which have force_fetch: true + elif not kwargs.get("force_fetch", False): + fetch_objs = [] + # iterate through targets + for target in target_objs: + try: + # get value of "force_fetch" property + dependencies = target["dependencies"] + # dependencies is still a list + for entry in dependencies: + force_fetch = entry["force_fetch"] + if force_fetch: + fetch_objs.append(target) + except KeyError: + # targets may have no "dependencies" or "force_fetch" key + continue + # fetch dependencies from targets with force_fetch set to true + if fetch_objs: + fetch_dependencies(output_path, fetch_objs, dep_cache_dir, True, pool) + + +class InventoryError(KapitanError): + """inventory error""" + + pass + + +class InventoryValidationError(InventoryError): + """inventory validation error""" + + pass diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py index ccb6f3a92..3a4fe256c 100644 --- a/kapitan/inventory/reclass.py +++ b/kapitan/inventory/reclass.py @@ -13,7 +13,7 @@ class ReclassInventory(Inventory): - def get_parameters(self, ignore_class_notfound=False) -> dict: + def get_parameters(self, ignore_class_notfound: bool = False) -> dict: """ Runs a reclass inventory in inventory_path (same output as running ./reclass.py -b inv_base_uri/ --inventory) @@ -36,13 +36,25 @@ def get_parameters(self, ignore_class_notfound=False) -> dict: class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - return _reclass.inventory() + return {name: node["parameters"] for name, node in _reclass.inventory()["nodes"].items()} except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") else: logger.error(f"Inventory reclass error: {e.message}") raise InventoryError(e.message) + + def get_targets(self, target_names: list[str]) -> dict: + targets = {} + + for target_name in target_names: + target = self.inventory.get(target_name) + if not target: + raise InventoryError(f"target '{target_name}' not found") + + targets[target_name] = target + + return targets def get_config(self) -> dict: @@ -65,13 +77,13 @@ def get_config(self) -> dict: if os.path.isfile(cfg_file): with open(cfg_file, "r") as fp: config = yaml.load(fp.read(), Loader=YamlLoader) - logger.debug("Using reclass inventory config at: {}".format(cfg_file)) + logger.debug(f"Using reclass inventory config at: {cfg_file}") if config: # set attributes, take default values if not present for key, value in config.items(): reclass_config[key] = value else: - logger.debug("{}: Empty config file. Using reclass inventory config defaults".format(cfg_file)) + logger.debug(f"Reclass config: Empty config file at {cfg_file}. Using reclass inventory config defaults") else: logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") diff --git a/kapitan/refs/cmd_parser.py b/kapitan/refs/cmd_parser.py index f6c1e22f1..f7f688c72 100644 --- a/kapitan/refs/cmd_parser.py +++ b/kapitan/refs/cmd_parser.py @@ -16,7 +16,7 @@ from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints from kapitan.refs.secrets.vaultkv import VaultSecret from kapitan.refs.secrets.vaulttransit import VaultTransit -from kapitan.resources import inventory_reclass +from kapitan.resources import Inventory from kapitan.utils import fatal_error, search_target_token_paths logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -95,7 +95,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -123,7 +123,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -152,7 +152,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -196,7 +196,7 @@ def ref_write(args, ref_controller): vault_params = {} encoding = "original" if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -251,7 +251,7 @@ def ref_write(args, ref_controller): _data = data.encode() vault_params = {} if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -323,7 +323,7 @@ def secret_update(args, ref_controller): for name in args.recipients ] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -351,7 +351,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("gkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -377,7 +377,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("azkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -403,7 +403,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("awskms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -460,7 +460,7 @@ def secret_update_validate(args, ref_controller): "Validate and/or update target secrets" # update gpg recipients/gkms/awskms key for all secrets in secrets_path # use --refs-path to set scanning path - inv = inventory_reclass(args.inventory_path) + inv = Inventory(args.inventory_path).inventory targets = set(inv["nodes"].keys()) secrets_path = os.path.abspath(args.refs_path) target_token_paths = search_target_token_paths(secrets_path, targets) diff --git a/kapitan/resources.py b/kapitan/resources.py index e568d6618..da8e9dfa9 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -17,23 +17,15 @@ from functools import partial import jsonschema +from kapitan.inventory import Inventory, ReclassInventory import kapitan.cached as cached import yaml from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -import reclass -import reclass.core -from reclass.errors import NotFoundError, ReclassException - logger = logging.getLogger(__name__) -try: - from yaml import CSafeLoader as YamlLoader -except ImportError: - from yaml import SafeLoader as YamlLoader - JSONNET_CACHE = {} @@ -280,17 +272,20 @@ def inventory(search_paths, target, inventory_path=None): if not inv_path_exists: raise InventoryError(f"Inventory not found in search paths: {search_paths}") + + inv = ReclassInventory(full_inv_path).inventory + + if target: + return {"parameters": inv[target]} + + return {name: {"parameters": params} for name, params in inv.items()} - if target is None: - return inventory_reclass(full_inv_path)["nodes"] - - return inventory_reclass(full_inv_path)["nodes"][target] def generate_inventory(args): try: - inv = inventory_reclass(args.inventory_path) - if args.target_name != "": + inv = Inventory.get().inventory + if not args.target_name: inv = inv["nodes"][args.target_name] if args.pattern != "": pattern = args.pattern.split(".") @@ -304,68 +299,3 @@ def generate_inventory(args): if not isinstance(e, KapitanError): logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") sys.exit(1) - - -def inventory_reclass(inventory_path, ignore_class_notfound=False): - """ - Runs a reclass inventory in inventory_path - (same output as running ./reclass.py -b inv_base_uri/ --inventory) - Will attempt to read reclass config from 'reclass-config.yml' otherwise - it will failback to the default config. - Returns a reclass style dictionary - - Does not throw errors if a class is not found while --fetch flag is enabled - """ - # if inventory is already cached theres nothing to do - if cached.inv: - return cached.inv - - # set default values initially - reclass_config = reclass_config_defaults = { - "storage_type": "yaml_fs", - "inventory_base_uri": inventory_path, - "nodes_uri": "targets", - "classes_uri": "classes", - "compose_node_name": False, - "allow_none_override": True, - "ignore_class_notfound": ignore_class_notfound, # false by default - } - - # get reclass config from file 'inventory/reclass-config.yml' - cfg_file = os.path.join(inventory_path, "reclass-config.yml") - if os.path.isfile(cfg_file): - with open(cfg_file, "r") as fp: - config = yaml.load(fp.read(), Loader=YamlLoader) - logger.debug("Using reclass inventory config at: {}".format(cfg_file)) - if config: - # set attributes, take default values if not present - for key, value in config.items(): - reclass_config[key] = value - else: - logger.debug("{}: Empty config file. Using reclass inventory config defaults".format(cfg_file)) - else: - logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") - - # normalise relative nodes_uri and classes_uri paths - for uri in ("nodes_uri", "classes_uri"): - reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri])) - - try: - storage = reclass.get_storage( - reclass_config["storage_type"], - reclass_config["nodes_uri"], - reclass_config["classes_uri"], - reclass_config["compose_node_name"], - ) - class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) - _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - - cached.inv = _reclass.inventory() - except ReclassException as e: - if isinstance(e, NotFoundError): - logger.error("Inventory reclass error: inventory not found") - else: - logger.error("Inventory reclass error: %s", e.message) - raise InventoryError(e.message) - - return cached.inv diff --git a/kapitan/targets.py b/kapitan/targets.py index c5e18e36f..fe5264644 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -23,6 +23,7 @@ from kapitan import cached, defaults from kapitan.dependency_manager.base import fetch_dependencies +from kapitan.remoteinventory.fetch import fetch_inventories, list_sources from kapitan.errors import CompileError, InventoryError, KapitanError from kapitan.inputs.copy import Copy from kapitan.inputs.external import External @@ -31,9 +32,8 @@ from kapitan.inputs.jsonnet import Jsonnet from kapitan.inputs.kadet import Kadet from kapitan.inputs.remove import Remove -from kapitan.remoteinventory.fetch import fetch_inventories, list_sources -from kapitan.resources import inventory_reclass -from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache +from kapitan.inventory import Inventory, ReclassInventory +from kapitan.utils import hashable_lru_cache from kapitan.validator.kubernetes_validator import KubernetesManifestValidator logger = logging.getLogger(__name__) @@ -42,6 +42,7 @@ def compile_targets( inventory_path, search_paths, output_path, parallel, targets, labels, ref_controller, **kwargs ): + _ = ReclassInventory(inventory_path) """ Searches and loads target files, and runs compile_target() on a multiprocessing pool with parallel number of processes. @@ -54,30 +55,10 @@ def compile_targets( temp_compile_path = os.path.join(temp_path, "compiled") dep_cache_dir = temp_path - updated_targets = targets - try: - updated_targets = search_targets(inventory_path, targets, labels) - except CompileError as e: - logger.error(e) - sys.exit(1) - - # If --cache is set - if kwargs.get("cache"): - additional_cache_paths = kwargs.get("cache_paths") - generate_inv_cache_hashes(inventory_path, targets, additional_cache_paths) - # to cache fetched dependencies and inventories - dep_cache_dir = os.path.join(output_path, ".dependency_cache") - os.makedirs(dep_cache_dir, exist_ok=True) - - if not targets: - updated_targets = changed_targets(inventory_path, output_path) - logger.debug("Changed targets since last compilation: %s", updated_targets) - if len(updated_targets) == 0: - logger.info("No changes since last compilation.") - return - pool = multiprocessing.Pool(parallel) + updated_targets = Inventory.get().search_targets() + try: rendering_start = time.time() @@ -94,10 +75,10 @@ def compile_targets( if fetch: # skip classes that are not yet available - target_objs = load_target_inventory(inventory_path, updated_targets, ignore_class_notfound=True) + target_objs = load_target_inventory(updated_targets) else: # ignore_class_notfound = False by default - target_objs = load_target_inventory(inventory_path, updated_targets) + target_objs = load_target_inventory(updated_targets) # append "compiled" to output_path so we can safely overwrite it compile_path = os.path.join(output_path, "compiled") @@ -105,49 +86,6 @@ def compile_targets( if not target_objs: raise CompileError("Error: no targets found") - # fetch inventory - if fetch: - # new_source checks for new sources in fetched inventory items - new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) - while new_sources: - fetch_inventories( - inventory_path, - target_objs, - dep_cache_dir, - force_fetch, - pool, - ) - cached.reset_inv() - target_objs = load_target_inventory( - inventory_path, updated_targets, ignore_class_notfound=True - ) - cached.inv_sources.update(new_sources) - new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) - # reset inventory cache and load target objs to check for missing classes - cached.reset_inv() - target_objs = load_target_inventory(inventory_path, updated_targets, ignore_class_notfound=False) - # fetch dependencies - if fetch: - fetch_dependencies(output_path, target_objs, dep_cache_dir, force_fetch, pool) - # fetch targets which have force_fetch: true - elif not kwargs.get("force_fetch", False): - fetch_objs = [] - # iterate through targets - for target in target_objs: - try: - # get value of "force_fetch" property - dependencies = target["dependencies"] - # dependencies is still a list - for entry in dependencies: - force_fetch = entry["force_fetch"] - if force_fetch: - fetch_objs.append(target) - except KeyError: - # targets may have no "dependencies" or "force_fetch" key - continue - # fetch dependencies from targets with force_fetch set to true - if fetch_objs: - fetch_dependencies(output_path, fetch_objs, dep_cache_dir, True, pool) logger.info("Rendered inventory (%.2fs)", time.time() - rendering_start) @@ -163,14 +101,14 @@ def compile_targets( # compile_target() returns None on success # so p is only not None when raising an exception - [p.get() for p in pool.imap_unordered(worker, target_objs) if p] + [p.get() for p in pool.imap_unordered(worker, target_objs.values()) if p] os.makedirs(compile_path, exist_ok=True) # if '-t' is set on compile or only a few changed, only override selected targets if updated_targets: - for target in target_objs: - path = target["target_full_path"] + for target in target_objs.values(): + path = target["_reclass_"]["name"]["path"] compile_path_target = os.path.join(compile_path, path) temp_path_target = os.path.join(temp_compile_path, path) @@ -185,17 +123,6 @@ def compile_targets( shutil.copytree(temp_compile_path, compile_path) logger.debug("Copied %s into %s", temp_compile_path, compile_path) - # validate the compiled outputs - if kwargs.get("validate", False): - validate_map = create_validate_mapping(target_objs, compile_path) - worker = partial( - schema_validate_kubernetes_output, - cache_dir=kwargs.get("schemas_path", "./schemas"), - ) - [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p] - - # Save inventory and folders cache - save_inv_cache(compile_path, targets) pool.close() except ReclassException as e: @@ -225,239 +152,51 @@ def compile_targets( logger.debug("Removed %s", temp_path) -def generate_inv_cache_hashes(inventory_path, targets, cache_paths): - """ - generates the hashes for the inventory per target and jsonnet/jinja2 folders for caching purposes - struct: { - inventory: - : - classes: - parameters: - folder: - components: - docs: - lib: - scripts: - ... - } - """ - inv = inventory_reclass(inventory_path) - cached.inv_cache = {} - cached.inv_cache["inventory"] = {} - cached.inv_cache["folder"] = {} - - if targets: - for target in targets: - try: - cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target]["classes"] = dictionary_hash( - inv["nodes"][target]["classes"] - ) - cached.inv_cache["inventory"][target]["parameters"] = dictionary_hash( - inv["nodes"][target]["parameters"] - ) - except KeyError: - raise CompileError("target not found: {}".format(target)) - else: - for target in inv["nodes"]: - cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target]["classes"] = dictionary_hash( - inv["nodes"][target]["classes"] - ) - cached.inv_cache["inventory"][target]["parameters"] = dictionary_hash( - inv["nodes"][target]["parameters"] - ) - - compile_obj = inv["nodes"][target]["parameters"]["kapitan"]["compile"] - for obj in compile_obj: - for input_path in obj["input_paths"]: - base_folder = os.path.dirname(input_path).split("/")[0] - if base_folder == "": - base_folder = os.path.basename(input_path).split("/")[0] - - if base_folder not in cached.inv_cache["folder"].keys(): - if os.path.exists(base_folder) and os.path.isdir(base_folder): - cached.inv_cache["folder"][base_folder] = directory_hash(base_folder) - - # Cache additional folders set by --cache-paths - for path in cache_paths: - if path not in cached.inv_cache["folder"].keys(): - if os.path.exists(path) and os.path.isdir(path): - cached.inv_cache["folder"][path] = directory_hash(path) - - # Most commonly changed but not referenced in input_paths - for common in ("lib", "vendor", "secrets"): - if common not in cached.inv_cache["folder"].keys(): - if os.path.exists(common) and os.path.isdir(common): - cached.inv_cache["folder"][common] = directory_hash(common) - - -def changed_targets(inventory_path, output_path): - """returns a list of targets that have changed since last compilation""" - targets = [] - inv = inventory_reclass(inventory_path) - - saved_inv_cache = None - saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache") - if os.path.exists(saved_inv_cache_path): - with open(saved_inv_cache_path, "r") as f: - try: - saved_inv_cache = yaml.safe_load(f) - except Exception: - raise CompileError("Failed to load kapitan cache: %s", saved_inv_cache_path) - - targets_list = list(inv["nodes"]) - - # If .kapitan_cache doesn't exist or failed to load, recompile all targets - if not saved_inv_cache: - return targets_list - else: - for key, hash in cached.inv_cache["folder"].items(): - try: - if hash != saved_inv_cache["folder"][key]: - logger.debug("%s folder hash changed, recompiling all targets", key) - return targets_list - except KeyError: - # Errors usually occur when saved_inv_cache doesn't contain a new folder - # Recompile anyway to be safe - return targets_list - - for target in targets_list: - try: - if ( - cached.inv_cache["inventory"][target]["classes"] - != saved_inv_cache["inventory"][target]["classes"] - ): - logger.debug("classes hash changed in %s, recompiling", target) - targets.append(target) - elif ( - cached.inv_cache["inventory"][target]["parameters"] - != saved_inv_cache["inventory"][target]["parameters"] - ): - logger.debug("parameters hash changed in %s, recompiling", target) - targets.append(target) - except KeyError: - # Errors usually occur when saved_inv_cache doesn't contain a new target - # Recompile anyway to be safe - targets.append(target) - - return targets - - -def save_inv_cache(compile_path, targets): - """save the cache to .kapitan_cache for inventories per target and folders""" - if cached.inv_cache: - inv_cache_path = os.path.join(compile_path, ".kapitan_cache") - # If only some targets were selected (-t), overwride only their inventory - if targets: - saved_inv_cache = None - try: - with open(inv_cache_path, "r") as f: - saved_inv_cache = yaml.safe_load(f) - except Exception: - pass - - if saved_inv_cache: - if "inventory" not in saved_inv_cache: - saved_inv_cache["inventory"] = {} - else: - saved_inv_cache = {} - saved_inv_cache["inventory"] = {} - - for target in targets: - if target not in saved_inv_cache["inventory"]: - saved_inv_cache["inventory"][target] = {} - - saved_inv_cache["inventory"][target]["classes"] = cached.inv_cache["inventory"][target][ - "classes" - ] - saved_inv_cache["inventory"][target]["parameters"] = cached.inv_cache["inventory"][target][ - "parameters" - ] - - with open(inv_cache_path, "w") as f: - logger.debug("Saved .kapitan_cache for targets: %s", targets) - yaml.dump(saved_inv_cache, stream=f, default_flow_style=False) - - else: - with open(inv_cache_path, "w") as f: - logger.debug("Saved .kapitan_cache") - yaml.dump(cached.inv_cache, stream=f, default_flow_style=False) - - -def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): +def load_target_inventory(targets): """returns a list of target objects from the inventory""" - target_objs = [] - inv = inventory_reclass(inventory_path, ignore_class_notfound) # if '-t' is set on compile, only loop through selected targets - if targets: - targets_list = targets + if not targets: + return Inventory.get().inventory else: - targets_list = inv["nodes"] - - for target_name in targets_list: - try: - inv_target = inv["nodes"][target_name] - target_obj = inv_target["parameters"]["kapitan"] - # check if parameters.kapitan is empty - if not target_obj: - raise InventoryError( - "InventoryError: {}: parameters.kapitan has no assignment".format(target_name) - ) - target_obj["target_full_path"] = inv_target["parameters"]["_reclass_"]["name"]["path"] - require_compile = not ignore_class_notfound - valid_target_obj(target_obj, require_compile) - validate_matching_target_name(target_name, target_obj, inventory_path) - logger.debug("load_target_inventory: found valid kapitan target %s", target_name) - target_objs.append(target_obj) - except KeyError: - logger.debug("load_target_inventory: target %s has no kapitan compile obj", target_name) - pass - - return target_objs - - -def search_targets(inventory_path, targets, labels): - """returns a list of targets where the labels match, otherwise just return the original targets""" - if not labels: - return targets + return Inventory.get().get_targets(targets) - try: - labels_dict = dict(label.split("=") for label in labels) - except ValueError: - raise CompileError( - "Compile error: Failed to parse labels, should be formatted like: kapitan compile -l env=prod app=example" - ) - targets_found = [] - inv = inventory_reclass(inventory_path) +# def filter_targets_to_compile(targets, labels): +# """returns a list of targets where the labels match, otherwise just return the original targets""" +# if not labels: +# return targets - for target_name in inv["nodes"]: - matched_all_labels = False - for label, value in labels_dict.items(): - try: - if inv["nodes"][target_name]["parameters"]["kapitan"]["labels"][label] == value: - matched_all_labels = True - continue - except KeyError: - logger.debug("search_targets: label %s=%s didn't match target %s", label, value, target_name) +# try: +# labels_dict = dict(label.split("=") for label in labels) +# except ValueError: +# raise CompileError( +# "Failed to parse labels. Your command should be formatted like: kapitan compile --labels env=prod app=example" +# ) - matched_all_labels = False - break +# for target_name in targets: +# matched_all_labels = False +# for label, value in labels_dict.items(): +# try: +# if inv["nodes"][target_name]["parameters"]["kapitan"]["labels"][label] == value: +# matched_all_labels = True +# continue +# except KeyError: +# logger.debug("search_targets: label %s=%s didn't match target %s", label, value, target_name) - if matched_all_labels: - targets_found.append(target_name) +# matched_all_labels = False +# break - if len(targets_found) == 0: - raise CompileError("No targets found with labels: {}".format(labels)) +# if matched_all_labels: +# targets_found.append(target_name) - return targets_found +# return targets_found def compile_target(target_obj, search_paths, compile_path, ref_controller, globals_cached=None, **kwargs): """Compiles target_obj and writes to compile_path""" start = time.time() + target_obj = target_obj["kapitan"] compile_objs = target_obj["compile"] ext_vars = target_obj["vars"] target_name = ext_vars["target"] @@ -500,7 +239,7 @@ def compile_target(target_obj, search_paths, compile_path, ref_controller, globa input_compiler.make_compile_dirs(target_name, output_path, **kwargs) input_compiler.compile_obj(comp_obj, ext_vars, **kwargs) - logger.info("Compiled %s (%.2fs)", target_obj["target_full_path"], time.time() - start) + logger.info(f"Compiled target ({time.time() - start:.2f}s)") @hashable_lru_cache @@ -762,92 +501,3 @@ def validate_matching_target_name(target_filename, target_obj, inventory_path): "Target name should match the name of the target yml file in inventory" ) raise InventoryError(error_message) - - -def schema_validate_compiled(args): - """ - validates compiled output according to schemas specified in the inventory - """ - if not os.path.isdir(args.compiled_path): - logger.error("compiled-path %s not found", args.compiled_path) - sys.exit(1) - - if not os.path.isdir(args.schemas_path): - os.makedirs(args.schemas_path) - logger.info("created schema-cache-path at %s", args.schemas_path) - - worker = partial(schema_validate_kubernetes_output, cache_dir=args.schemas_path) - pool = multiprocessing.Pool(args.parallelism) - - try: - target_objs = load_target_inventory(args.inventory_path, args.targets) - validate_map = create_validate_mapping(target_objs, args.compiled_path) - - [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p] - pool.close() - - except ReclassException as e: - if isinstance(e, NotFoundError): - logger.error("Inventory reclass error: inventory not found") - else: - logger.error("Inventory reclass error: %s", e.message) - raise InventoryError(e.message) - except Exception as e: - pool.terminate() - logger.debug("Validate pool terminated") - # only print traceback for errors we don't know about - if not isinstance(e, KapitanError): - logger.exception("Unknown (Non-Kapitan) Error occured") - - logger.error("\n") - logger.error(e) - sys.exit(1) - finally: - # always wait for other worker processes to terminate - pool.join() - - -def create_validate_mapping(target_objs, compiled_path): - """ - creates mapping of (kind, version) tuple to output_paths across different targets - this is required to avoid redundant schema fetch when multiple targets use the same schema for validation - """ - validate_files_map = defaultdict(list) - for target_obj in target_objs: - target_name = target_obj["vars"]["target"] - if "validate" not in target_obj: - logger.debug( - "target '%s' does not have 'validate' parameter in inventory. skipping", - target_name, - ) - continue - - for validate_item in target_obj["validate"]: - validate_type = validate_item["type"] - if validate_type == "kubernetes": - kind_version_pair = ( - validate_item["kind"], - validate_item.get("version", defaults.DEFAULT_KUBERNETES_VERSION), - ) - for output_path in validate_item["output_paths"]: - full_output_path = os.path.join(compiled_path, target_name, output_path) - if not os.path.isfile(full_output_path): - logger.warning( - "%s does not exist for target '%s'. skipping", output_path, target_name - ) - continue - validate_files_map[kind_version_pair].append(full_output_path) - else: - logger.warning("type %s is not supported for validation. skipping", validate_type) - - return validate_files_map - - -def schema_validate_kubernetes_output(validate_data, cache_dir): - """ - validates given files according to kubernetes manifest schemas - schemas are cached from/to cache_dir - validate_data must be of structure ((kind, version), validate_files) - """ - (kind, version), validate_files = validate_data - KubernetesManifestValidator(cache_dir).validate(validate_files, kind=kind, version=version) diff --git a/tests/test_compile.py b/tests/test_compile.py index 5196ae99b..64b14694e 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -20,7 +20,7 @@ from kapitan.utils import directory_hash from kapitan.cached import reset_cache from kapitan.targets import validate_matching_target_name -from kapitan.resources import inventory_reclass +from kapitan.inventory import Inventory from kapitan.errors import InventoryError @@ -175,7 +175,7 @@ def test_compile_not_matching_targets(self): def test_compile_vars_target_missing(self): inventory_path = "inventory" target_filename = "minikube-es" - target_obj = inventory_reclass(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"] + target_obj = Inventory(inventory_path).inventory["nodes"][target_filename]["parameters"]["kapitan"] # delete vars.target del target_obj["vars"]["target"] From 87ce2e0ce6319f9e9a76302cea8d6bcba4bbc47e Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 13 Dec 2023 14:50:01 +0100 Subject: [PATCH 04/12] fix: remove unwanted changes --- kapitan/cli.py | 4 ++-- kapitan/inputs/base.py | 15 --------------- kapitan/inventory/reclass.py | 2 +- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 20e776cb8..d1d81ee8b 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -25,7 +25,7 @@ from kapitan.refs.base import RefController, Revealer from kapitan.refs.cmd_parser import handle_refs_command from kapitan.resources import generate_inventory, resource_callbacks, search_imports -from kapitan.targets import compile_targets +from kapitan.targets import compile_targets, schema_validate_compiled from kapitan.utils import check_version, from_dot_kapitan, searchvar from kapitan.version import DESCRIPTION, PROJECT_NAME, VERSION @@ -584,7 +584,7 @@ def build_parser(): aliases=["v"], help="validates the compile output against schemas as specified in inventory", ) - # validate_parser.set_defaults(func=schema_validate_compiled, name="validate") + validate_parser.set_defaults(func=schema_validate_compiled, name="validate") validate_parser.add_argument( "--compiled-path", diff --git a/kapitan/inputs/base.py b/kapitan/inputs/base.py index b8f8a95d1..711eff26f 100644 --- a/kapitan/inputs/base.py +++ b/kapitan/inputs/base.py @@ -18,25 +18,10 @@ from kapitan.errors import CompileError, KapitanError from kapitan.refs.base import Revealer from kapitan.utils import PrettyDumper -from enum import Enum logger = logging.getLogger(__name__) -class InputTypes(Enum): - """ - All currently supported input types in the inventory for parameters.kapitan.compile - """ - - KADET = "kadet" - JSONNET = "jsonnet" - JINJA2 = "jinja2" - HELM = "helm" - EXTERNAL = "external" - COPY = "copy" - REMOVE = "remove" - - class InputType(object): def __init__(self, type_name, compile_path, search_paths, ref_controller): self.type_name = type_name diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py index 3a4fe256c..d827d6a7f 100644 --- a/kapitan/inventory/reclass.py +++ b/kapitan/inventory/reclass.py @@ -91,4 +91,4 @@ def get_config(self) -> dict: for uri in ("nodes_uri", "classes_uri"): reclass_config[uri] = os.path.normpath(os.path.join(self.inventory_path, reclass_config[uri])) - return reclass_config \ No newline at end of file + return reclass_config From dbd989c24a0feb943f07573e88974d3f8fc15bba Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 15 Dec 2023 11:06:20 +0100 Subject: [PATCH 05/12] lint: apply black --- kapitan/inventory/base.py | 6 +++--- kapitan/inventory/reclass.py | 6 +++--- kapitan/resources.py | 7 +++---- kapitan/targets.py | 1 - 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/kapitan/inventory/base.py b/kapitan/inventory/base.py index 70003307d..f177bd714 100644 --- a/kapitan/inventory/base.py +++ b/kapitan/inventory/base.py @@ -5,12 +5,12 @@ # # SPDX-License-Identifier: Apache-2.0 -from abc import ABC, abstractmethod -from dataclasses import dataclass import logging import os -from kapitan import cached +from abc import ABC, abstractmethod +from dataclasses import dataclass +from kapitan import cached from kapitan.errors import InventoryError, KapitanError logger = logging.getLogger(__name__) diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py index d827d6a7f..ba11c91ab 100644 --- a/kapitan/inventory/reclass.py +++ b/kapitan/inventory/reclass.py @@ -1,13 +1,13 @@ -from kapitan.inventory.base import Inventory -import os -import yaml import logging +import os import reclass import reclass.core +import yaml from reclass.errors import NotFoundError, ReclassException from kapitan.errors import InventoryError +from kapitan.inventory.base import Inventory logger = logging.getLogger(__name__) diff --git a/kapitan/resources.py b/kapitan/resources.py index da8e9dfa9..4a90c6ca3 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -272,14 +272,13 @@ def inventory(search_paths, target, inventory_path=None): if not inv_path_exists: raise InventoryError(f"Inventory not found in search paths: {search_paths}") - + inv = ReclassInventory(full_inv_path).inventory - + if target: return {"parameters": inv[target]} - - return {name: {"parameters": params} for name, params in inv.items()} + return {name: {"parameters": params} for name, params in inv.items()} def generate_inventory(args): diff --git a/kapitan/targets.py b/kapitan/targets.py index fe5264644..6c162cc9d 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -86,7 +86,6 @@ def compile_targets( if not target_objs: raise CompileError("Error: no targets found") - logger.info("Rendered inventory (%.2fs)", time.time() - rendering_start) worker = partial( From 33039e79ff5645b0ac28fbf580aa75e0c88a0d19 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 15 Dec 2023 11:41:23 +0100 Subject: [PATCH 06/12] feat: add cli argument group --- kapitan/cli.py | 21 ++++++++++-- kapitan/inventory/base.py | 63 ++---------------------------------- kapitan/inventory/reclass.py | 4 ++- kapitan/resources.py | 16 ++++++--- kapitan/targets.py | 2 +- 5 files changed, 36 insertions(+), 70 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index d1d81ee8b..e83cee203 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -102,6 +102,15 @@ def build_parser(): parser.add_argument("--version", action="version", version=VERSION) subparser = parser.add_subparsers(help="commands", dest="subparser_name") + inventory_backend_parser = argparse.ArgumentParser(add_help=False) + inventory_backend_group = inventory_backend_parser.add_argument_group("inventory_backend") + inventory_backend_group.add_argument( + "--reclass", + action="store_true", + default=from_dot_kapitan("inventory_backend", "reclass", False), + help="use reclass as inventory backend (default)", + ) + eval_parser = subparser.add_parser("eval", aliases=["e"], help="evaluate jsonnet file") eval_parser.add_argument("jsonnet_file", type=str) eval_parser.set_defaults(func=trigger_eval, name="eval") @@ -131,7 +140,9 @@ def build_parser(): help='set search paths, default is ["."]', ) - compile_parser = subparser.add_parser("compile", aliases=["c"], help="compile targets") + compile_parser = subparser.add_parser( + "compile", aliases=["c"], help="compile targets", parents=[inventory_backend_parser] + ) compile_parser.set_defaults(func=trigger_compile, name="compile") compile_parser.add_argument( @@ -326,7 +337,9 @@ def build_parser(): metavar="key=value", ) - inventory_parser = subparser.add_parser("inventory", aliases=["i"], help="show inventory") + inventory_parser = subparser.add_parser( + "inventory", aliases=["i"], help="show inventory", parents=[inventory_backend_parser] + ) inventory_parser.set_defaults(func=generate_inventory, name="inventory") inventory_parser.add_argument( @@ -414,7 +427,9 @@ def build_parser(): secrets_parser = subparser.add_parser("secrets", aliases=["s"], help="(DEPRECATED) please use refs") secrets_parser.set_defaults(func=print_deprecated_secrets_msg, name="secrets") - refs_parser = subparser.add_parser("refs", aliases=["r"], help="manage refs") + refs_parser = subparser.add_parser( + "refs", aliases=["r"], help="manage refs", parents=[inventory_backend_parser] + ) refs_parser.set_defaults(func=handle_refs_command, name="refs") refs_parser.add_argument( diff --git a/kapitan/inventory/base.py b/kapitan/inventory/base.py index f177bd714..e89abb7c7 100644 --- a/kapitan/inventory/base.py +++ b/kapitan/inventory/base.py @@ -45,34 +45,18 @@ def parameters(self) -> dict(): class Inventory(ABC): - _instance = None - - def __new__(cls, *args, **kwargs): - """ - create only one specific inventory instance (singleton) - """ - if Inventory._instance is None: - Inventory._instance = super().__new__(cls) - return Inventory._instance - - def __init__(self, inventory_path: str, compose_node_name: dict = False): + def __init__(self, inventory_path: str, ignore_class_not_found: bool = False): self.inventory_path = inventory_path self.targets_path = os.path.join(inventory_path, "targets") self.classes_path = os.path.join(inventory_path, "classes") # config - self.compose_node_name = compose_node_name + # self.compose_node_name = compose_node_name # used as cache for inventory self._targets = {} self._parameters = {} - @classmethod - def get(cls): - if cls._instance is None: - raise InventoryError("no type specified") - return cls._instance - @property def inventory(self) -> dict: if not self._parameters: @@ -128,49 +112,6 @@ def get_targets(self, target_names: list[str]) -> dict: """ raise NotImplementedError - def fetch_dependencies(self, fetch, force_fetch): - # fetch inventory - if fetch: - # new_source checks for new sources in fetched inventory items - new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) - while new_sources: - fetch_inventories( - inventory_path, - target_objs, - dep_cache_dir, - force_fetch, - pool, - ) - cached.reset_inv() - target_objs = load_target_inventory(updated_targets) - cached.inv_sources.update(new_sources) - new_sources = list(set(list_sources(target_objs)) - cached.inv_sources) - # reset inventory cache and load target objs to check for missing classes - cached.reset_inv() - target_objs = load_target_inventory(updated_targets) - # fetch dependencies - if fetch: - fetch_dependencies(output_path, target_objs, dep_cache_dir, force_fetch, pool) - # fetch targets which have force_fetch: true - elif not kwargs.get("force_fetch", False): - fetch_objs = [] - # iterate through targets - for target in target_objs: - try: - # get value of "force_fetch" property - dependencies = target["dependencies"] - # dependencies is still a list - for entry in dependencies: - force_fetch = entry["force_fetch"] - if force_fetch: - fetch_objs.append(target) - except KeyError: - # targets may have no "dependencies" or "force_fetch" key - continue - # fetch dependencies from targets with force_fetch set to true - if fetch_objs: - fetch_dependencies(output_path, fetch_objs, dep_cache_dir, True, pool) - class InventoryError(KapitanError): """inventory error""" diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py index ba11c91ab..5fd4ccf7e 100644 --- a/kapitan/inventory/reclass.py +++ b/kapitan/inventory/reclass.py @@ -35,8 +35,10 @@ def get_parameters(self, ignore_class_notfound: bool = False) -> dict: ) class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) + + # TODO: return only target: parameters --> {name: node["parameters"] for name, node in ["nodes"].items()} + return _reclass.inventory() - return {name: node["parameters"] for name, node in _reclass.inventory()["nodes"].items()} except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") diff --git a/kapitan/resources.py b/kapitan/resources.py index e0a09d0ba..53d00f60d 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -308,8 +308,16 @@ def get_inventory(inventory_path, ignore_class_notfound=False): if cached.inv: return cached.inv - logger.debug("Using reclass as inventory backend") - inv = inventory_reclass(inventory_path, ignore_class_notfound) + inventory_backend: Inventory = None - cached.inv = inv - return inv + # select inventory backend + if cached.args.get("inventory-backend") == "my-new-inventory": + logger.debug("Using my-new-inventory as inventory backend") + else: + logger.debug("Using reclass as inventory backend") + inventory_backend = ReclassInventory(inventory_path, ignore_class_notfound) + + inventory = inventory_backend.inventory + + cached.inv = inventory + return inventory diff --git a/kapitan/targets.py b/kapitan/targets.py index ed81c414f..4d06da5ec 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -850,4 +850,4 @@ def schema_validate_kubernetes_output(validate_data, cache_dir): validate_data must be of structure ((kind, version), validate_files) """ (kind, version), validate_files = validate_data - KubernetesManifestValidator(cache_dir).validate(validate_files, kind=kind, version=version) \ No newline at end of file + KubernetesManifestValidator(cache_dir).validate(validate_files, kind=kind, version=version) From 671c3c9c249232459de5e550053bb06147a43a74 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Thu, 11 Jan 2024 11:59:17 +0100 Subject: [PATCH 07/12] feat: add reclass implementation to interface --- kapitan/inputs/jsonnet.py | 1 + kapitan/inventory/__init__.py | 4 +- kapitan/inventory/base.py | 125 --------------------------- kapitan/inventory/inv_reclass.py | 102 ++++++++++++++++++++++ kapitan/inventory/inventory.py | 118 +++++++++++++++++++++++++ kapitan/inventory/reclass.py | 96 -------------------- kapitan/refs/cmd_parser.py | 30 +++---- kapitan/refs/secrets/awskms.py | 9 +- kapitan/refs/secrets/azkms.py | 19 ++-- kapitan/refs/secrets/gkms.py | 13 ++- kapitan/refs/secrets/gpg.py | 9 +- kapitan/refs/secrets/vaultkv.py | 10 +-- kapitan/refs/secrets/vaulttransit.py | 10 +-- kapitan/resources.py | 39 +++++---- kapitan/targets.py | 67 +++++--------- 15 files changed, 313 insertions(+), 339 deletions(-) delete mode 100644 kapitan/inventory/base.py create mode 100644 kapitan/inventory/inv_reclass.py create mode 100644 kapitan/inventory/inventory.py delete mode 100644 kapitan/inventory/reclass.py diff --git a/kapitan/inputs/jsonnet.py b/kapitan/inputs/jsonnet.py index cc101ef15..229cb1d3a 100644 --- a/kapitan/inputs/jsonnet.py +++ b/kapitan/inputs/jsonnet.py @@ -10,6 +10,7 @@ import os import sys +from kapitan import cached from kapitan.errors import CompileError from kapitan.inputs.base import CompiledFile, InputType from kapitan.resources import resource_callbacks, search_imports diff --git a/kapitan/inventory/__init__.py b/kapitan/inventory/__init__.py index 266ca4f5e..16268e391 100644 --- a/kapitan/inventory/__init__.py +++ b/kapitan/inventory/__init__.py @@ -1,2 +1,2 @@ -from .base import Inventory -from .reclass import ReclassInventory +from .inv_reclass import ReclassInventory +from .inventory import Inventory diff --git a/kapitan/inventory/base.py b/kapitan/inventory/base.py deleted file mode 100644 index e89abb7c7..000000000 --- a/kapitan/inventory/base.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2023 The Kapitan Authors -# SPDX-FileCopyrightText: 2023 The Kapitan Authors -# -# SPDX-License-Identifier: Apache-2.0 - -import logging -import os -from abc import ABC, abstractmethod -from dataclasses import dataclass - -from kapitan import cached -from kapitan.errors import InventoryError, KapitanError - -logger = logging.getLogger(__name__) - - -@dataclass -class InventoryTarget(dict): - targets_path: str - - def __init__(self, name: str, path: str): - self.name = name - self.path = path - - # compose node name - self.composed_name = os.path.splitext(path)[0].replace(path + os.sep, "").replace("/", ".") - - @property - def kapitan(self) -> dict(): - kapitan_spec = self.get("kapitan") - if not kapitan_spec: - raise InventoryValidationError("parameters.kapitan is empty") - - return kapitan_spec - - @property - def parameters(self) -> dict(): - parameters = self.get("parameters") - if not parameters: - raise InventoryValidationError("parameters is empty") - - return parameters - - -class Inventory(ABC): - def __init__(self, inventory_path: str, ignore_class_not_found: bool = False): - self.inventory_path = inventory_path - self.targets_path = os.path.join(inventory_path, "targets") - self.classes_path = os.path.join(inventory_path, "classes") - - # config - # self.compose_node_name = compose_node_name - - # used as cache for inventory - self._targets = {} - self._parameters = {} - - @property - def inventory(self) -> dict: - if not self._parameters: - self._parameters = self.get_parameters() - cached.inv = self._parameters - return self._parameters - - @abstractmethod - def get_parameters(self, ignore_class_notfound: bool = False) -> dict: - """ - create the inventory depending which backend gets used - """ - raise NotImplementedError - - def search_targets(self) -> dict: - """ - look for targets at '/targets/' and return targets - """ - for root, _, files in os.walk(self.targets_path): - for file in files: - # split file extension and check if yml/yaml - path = os.path.join(root, file) - name, ext = os.path.splitext(file) - if ext not in (".yml", ".yaml"): - logger.debug(f"{file}: targets have to be .yml or .yaml files.") - continue - - # initialize target - target = InventoryTarget(name, path) - if self.compose_node_name: - target.name = target.composed_name - - # check for same name - if self._targets.get(target.name): - raise InventoryError( - f"Conflicting targets {target.name}: {target.path} and {self._targets[target.name].path}" - ) - - self._targets[target.name] = target - - return self._targets - - def get_target(self, target_name: str) -> dict: - """ - get parameters for a specific target - """ - return self.get_targets([target_name]) - - @abstractmethod - def get_targets(self, target_names: list[str]) -> dict: - """ - get parameters for multiple targets - """ - raise NotImplementedError - - -class InventoryError(KapitanError): - """inventory error""" - - pass - - -class InventoryValidationError(InventoryError): - """inventory validation error""" - - pass diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py new file mode 100644 index 000000000..1ef113d24 --- /dev/null +++ b/kapitan/inventory/inv_reclass.py @@ -0,0 +1,102 @@ +import logging +import os + +import reclass +import reclass.core +import yaml +from reclass.errors import NotFoundError, ReclassException + +from kapitan.errors import InventoryError + +from .inventory import Inventory + +logger = logging.getLogger(__name__) + + +class ReclassInventory(Inventory): + + def render_targets(self, targets: list = None, ignore_class_notfound: bool = False): + """ + Runs a reclass inventory in inventory_path + (same output as running ./reclass.py -b inv_base_uri/ --inventory) + Will attempt to read reclass config from 'reclass-config.yml' otherwise + it will fall back to the default config. + Returns a reclass style dictionary + + Does not throw errors if a class is not found while ignore_class_notfound is specified + """ + reclass_config = get_reclass_config(self.inventory_path) + reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound) + + try: + storage = reclass.get_storage( + reclass_config["storage_type"], + reclass_config["nodes_uri"], + reclass_config["classes_uri"], + reclass_config["compose_node_name"], + ) + class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) + _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) + rendered_inventory = _reclass.inventory() + + # store parameters and classes + for name, rendered_target in rendered_inventory["nodes"].items(): + self.targets[name].parameters = rendered_target["parameters"] + + except ReclassException as e: + if isinstance(e, NotFoundError): + logger.error("Inventory reclass error: inventory not found") + else: + logger.error(f"Inventory reclass error: {e.message}") + raise InventoryError(e.message) + + def get_targets(self, target_names: list[str]) -> dict: + + for target_name in target_names: + target = self.targets.get(target_name) + if not target: + raise InventoryError(f"target '{target_name}' not found") + + if not target.parameters: + # reclass has no optimization for rendering only some specific targets, + # so we have to render the whole inventory + self.render_targets() + + return {name: target.parameters for name, target in self.targets.items() if name in target_names} + + +def get_reclass_config(inventory_path: str) -> dict: + # set default values initially + reclass_config = { + "storage_type": "yaml_fs", + "inventory_base_uri": inventory_path, + "nodes_uri": "targets", + "classes_uri": "classes", + "compose_node_name": False, + "allow_none_override": True, + } + try: + from yaml import CSafeLoader as YamlLoader + except ImportError: + from yaml import SafeLoader as YamlLoader + + # get reclass config from file 'inventory/reclass-config.yml' + cfg_file = os.path.join(inventory_path, "reclass-config.yml") + if os.path.isfile(cfg_file): + with open(cfg_file, "r") as fp: + config = yaml.load(fp.read(), Loader=YamlLoader) + logger.debug(f"Using reclass inventory config at: {cfg_file}") + if config: + # set attributes, take default values if not present + for key, value in config.items(): + reclass_config[key] = value + else: + logger.debug(f"Reclass config: Empty config file at {cfg_file}. Using reclass inventory config defaults") + else: + logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") + + # normalise relative nodes_uri and classes_uri paths + for uri in ("nodes_uri", "classes_uri"): + reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri])) + + return reclass_config diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py new file mode 100644 index 000000000..4000ea08c --- /dev/null +++ b/kapitan/inventory/inventory.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The Kapitan Authors +# SPDX-FileCopyrightText: 2023 The Kapitan Authors +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + +from kapitan.errors import KapitanError +from kapitan.reclass.reclass.values import item + +logger = logging.getLogger(__name__) + + +@dataclass +class InventoryTarget: + name: str + path: str + composed_name: str + parameters: dict = field(default_factory=dict) + + +class Inventory(ABC): + path: str = "inventory" + targets: dict[str, InventoryTarget] = {} + + def __init__(self, path: str = path, compose_target_name: bool = False): + self.inventory_path = path + self.targets_path = os.path.join(path, "targets") + self.classes_path = os.path.join(path, "classes") + + # config + self.compose_target_name = compose_target_name + + @property + def inventory(self) -> dict: + if not self.targets: + return self.render_targets() + return {name: target.parameters for name, target in self.targets.items()} + + @abstractmethod + def render_targets(self, targets: list = None, ignore_class_notfound: bool = False) -> dict: + """ + create the inventory depending on which backend gets used + """ + raise NotImplementedError + + def search_targets(self) -> dict: + """ + look for targets at '/targets/' and return targets without rendering parameters + """ + for root, dirs, files in os.walk(self.targets_path): + for file in files: + # split file extension and check if yml/yaml + path = os.path.join(root, file) + name, ext = os.path.splitext(file) + if ext not in (".yml", ".yaml"): + logger.debug(f"{file}: targets have to be .yml or .yaml files.") + continue + + # initialize target + composed_name = ( + os.path.splitext(os.path.relpath(path, self.targets_path))[0] + .replace(os.sep, ".") + .lstrip(".") + ) + target = InventoryTarget(name, path, composed_name) + if self.compose_target_name: + target.name = target.composed_name + + # check for same name + if self.targets.get(target.name): + raise InventoryError( + f"Conflicting targets {target.name}: {target.path} and {self.targets[target.name].path}" + ) + + self.targets[target.name] = target + + return self.targets + + def get_target(self, target_name: str) -> dict: + """ + helper function to get parameters for a specific target + """ + return self.get_targets([target_name])[target_name] + + @abstractmethod + def get_targets(self, target_names: list[str]) -> dict: + """ + helper function to get parameters for multiple targets + """ + raise NotImplementedError + + def __getitem__(self, key): + return self.inventory[key] + + +class InventoryError(KapitanError): + """inventory error""" + + pass + + +class InventoryValidationError(InventoryError): + """inventory validation error""" + + pass + + +class InvalidTargetError(InventoryError): + """inventory validation error""" + + pass diff --git a/kapitan/inventory/reclass.py b/kapitan/inventory/reclass.py deleted file mode 100644 index 5fd4ccf7e..000000000 --- a/kapitan/inventory/reclass.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging -import os - -import reclass -import reclass.core -import yaml -from reclass.errors import NotFoundError, ReclassException - -from kapitan.errors import InventoryError -from kapitan.inventory.base import Inventory - -logger = logging.getLogger(__name__) - -class ReclassInventory(Inventory): - - def get_parameters(self, ignore_class_notfound: bool = False) -> dict: - """ - Runs a reclass inventory in inventory_path - (same output as running ./reclass.py -b inv_base_uri/ --inventory) - Will attempt to read reclass config from 'reclass-config.yml' otherwise - it will failback to the default config. - Returns a reclass style dictionary - - Does not throw errors if a class is not found while --fetch flag is enabled - """ - reclass_config = self.get_config() - reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound) - - try: - storage = reclass.get_storage( - reclass_config["storage_type"], - reclass_config["nodes_uri"], - reclass_config["classes_uri"], - reclass_config["compose_node_name"], - ) - class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) - _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - - # TODO: return only target: parameters --> {name: node["parameters"] for name, node in ["nodes"].items()} - return _reclass.inventory() - - except ReclassException as e: - if isinstance(e, NotFoundError): - logger.error("Inventory reclass error: inventory not found") - else: - logger.error(f"Inventory reclass error: {e.message}") - raise InventoryError(e.message) - - def get_targets(self, target_names: list[str]) -> dict: - targets = {} - - for target_name in target_names: - target = self.inventory.get(target_name) - if not target: - raise InventoryError(f"target '{target_name}' not found") - - targets[target_name] = target - - return targets - - - def get_config(self) -> dict: - # set default values initially - reclass_config = { - "storage_type": "yaml_fs", - "inventory_base_uri": self.inventory_path, - "nodes_uri": "targets", - "classes_uri": "classes", - "compose_node_name": False, - "allow_none_override": True, - } - try: - from yaml import CSafeLoader as YamlLoader - except ImportError: - from yaml import SafeLoader as YamlLoader - - # get reclass config from file 'inventory/reclass-config.yml' - cfg_file = os.path.join(self.inventory_path, "reclass-config.yml") - if os.path.isfile(cfg_file): - with open(cfg_file, "r") as fp: - config = yaml.load(fp.read(), Loader=YamlLoader) - logger.debug(f"Using reclass inventory config at: {cfg_file}") - if config: - # set attributes, take default values if not present - for key, value in config.items(): - reclass_config[key] = value - else: - logger.debug(f"Reclass config: Empty config file at {cfg_file}. Using reclass inventory config defaults") - else: - logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") - - # normalise relative nodes_uri and classes_uri paths - for uri in ("nodes_uri", "classes_uri"): - reclass_config[uri] = os.path.normpath(os.path.join(self.inventory_path, reclass_config[uri])) - - return reclass_config diff --git a/kapitan/refs/cmd_parser.py b/kapitan/refs/cmd_parser.py index c95850e18..fff735b5e 100644 --- a/kapitan/refs/cmd_parser.py +++ b/kapitan/refs/cmd_parser.py @@ -2,17 +2,17 @@ import base64 import logging +import mimetypes import os import sys -import mimetypes -from kapitan.errors import KapitanError, RefHashMismatchError, RefError +from kapitan.errors import KapitanError, RefError, RefHashMismatchError from kapitan.refs.base import PlainRef, RefController, Revealer from kapitan.refs.base64 import Base64Ref from kapitan.refs.env import EnvRef from kapitan.refs.secrets.awskms import AWSKMSSecret -from kapitan.refs.secrets.gkms import GoogleKMSSecret from kapitan.refs.secrets.azkms import AzureKMSSecret +from kapitan.refs.secrets.gkms import GoogleKMSSecret from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints from kapitan.refs.secrets.vaultkv import VaultSecret from kapitan.refs.secrets.vaulttransit import VaultTransit @@ -66,7 +66,7 @@ def ref_write(args, ref_controller): recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -96,7 +96,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -124,7 +124,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -153,7 +153,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -197,7 +197,7 @@ def ref_write(args, ref_controller): encoding = "original" if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -252,7 +252,7 @@ def ref_write(args, ref_controller): vault_params = {} if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -324,7 +324,7 @@ def secret_update(args, ref_controller): ] if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -352,7 +352,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -378,7 +378,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -404,7 +404,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -461,13 +461,13 @@ def secret_update_validate(args, ref_controller): # update gpg recipients/gkms/awskms key for all secrets in secrets_path # use --refs-path to set scanning path inv = get_inventory(args.inventory_path) - targets = set(inv["nodes"].keys()) + targets = set(inv.targets.keys()) secrets_path = os.path.abspath(args.refs_path) target_token_paths = search_target_token_paths(secrets_path, targets) ret_code = 0 for target_name, token_paths in target_token_paths.items(): - kap_inv_params = inv["nodes"][target_name]["parameters"]["kapitan"] + kap_inv_params = inv.get_target(target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(target_name)) diff --git a/kapitan/refs/secrets/awskms.py b/kapitan/refs/secrets/awskms.py index 051639342..7e12908af 100644 --- a/kapitan/refs/secrets/awskms.py +++ b/kapitan/refs/secrets/awskms.py @@ -6,12 +6,13 @@ "awskms secrets module" import base64 + import boto3 -from kapitan.refs.base import RefError -from kapitan.refs.base64 import Base64Ref, Base64RefBackend from kapitan import cached from kapitan.errors import KapitanError +from kapitan.refs.base import RefError +from kapitan.refs.base64 import Base64Ref, Base64RefBackend class AWSKMSError(KapitanError): @@ -54,11 +55,11 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) + target_inv = cached.inv.get_target(target_name) if target_inv is None: raise ValueError("target_inv not set") - key = target_inv["parameters"]["kapitan"]["secrets"]["awskms"]["key"] + key = target_inv["kapitan"]["secrets"]["awskms"]["key"] return cls(data, key, **ref_params.kwargs) except KeyError: raise RefError("Could not create AWSKMSSecret: target_name missing") diff --git a/kapitan/refs/secrets/azkms.py b/kapitan/refs/secrets/azkms.py index 561f6d67b..c5e694945 100644 --- a/kapitan/refs/secrets/azkms.py +++ b/kapitan/refs/secrets/azkms.py @@ -1,17 +1,18 @@ "azkms secret module" -import os -import logging import base64 +import logging +import os from urllib.parse import urlparse -from azure.keyvault.keys.crypto import CryptographyClient, EncryptionAlgorithm -from azure.keyvault.keys import KeyClient + from azure.identity import DefaultAzureCredential +from azure.keyvault.keys import KeyClient +from azure.keyvault.keys.crypto import CryptographyClient, EncryptionAlgorithm -from kapitan.refs.base64 import Base64Ref, Base64RefBackend -from kapitan.refs.base import RefError from kapitan import cached from kapitan.errors import KapitanError +from kapitan.refs.base import RefError +from kapitan.refs.base64 import Base64Ref, Base64RefBackend logger = logging.getLogger(__name__) @@ -78,11 +79,9 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) - if target_inv is None: - raise ValueError("target_inv not set") + target_inv = cached.inv.get_target(target_name) - key = target_inv["parameters"]["kapitan"]["secrets"]["azkms"]["key"] + key = target_inv["kapitan"]["secrets"]["azkms"]["key"] return cls(data, key, **ref_params.kwargs) except KeyError: raise RefError("Could not create AzureKMSSecret: target_name missing") diff --git a/kapitan/refs/secrets/gkms.py b/kapitan/refs/secrets/gkms.py index b628c5225..7536c989f 100644 --- a/kapitan/refs/secrets/gkms.py +++ b/kapitan/refs/secrets/gkms.py @@ -6,14 +6,15 @@ "gkms secrets module" import base64 -import googleapiclient.discovery as gcloud import logging import warnings -from kapitan.refs.base64 import Base64Ref, Base64RefBackend -from kapitan.refs.base import RefError +import googleapiclient.discovery as gcloud + from kapitan import cached from kapitan.errors import KapitanError +from kapitan.refs.base import RefError +from kapitan.refs.base64 import Base64Ref, Base64RefBackend logger = logging.getLogger(__name__) @@ -65,11 +66,9 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) - if target_inv is None: - raise ValueError("target_inv not set") + target_inv = cached.inv.get_target(target_name) - key = target_inv["parameters"]["kapitan"]["secrets"]["gkms"]["key"] + key = target_inv["kapitan"]["secrets"]["gkms"]["key"] return cls(data, key, **ref_params.kwargs) except KeyError: raise RefError("Could not create GoogleKMSSecret: target_name missing") diff --git a/kapitan/refs/secrets/gpg.py b/kapitan/refs/secrets/gpg.py index eedbf4fee..bde0d7f2d 100644 --- a/kapitan/refs/secrets/gpg.py +++ b/kapitan/refs/secrets/gpg.py @@ -10,6 +10,7 @@ import time import gnupg + from kapitan import cached from kapitan.errors import KapitanError from kapitan.refs.base import RefError @@ -75,16 +76,14 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) - if target_inv is None: - raise ValueError("target_inv not set") + target_inv = cached.inv.get_target(target_name) - if "secrets" not in target_inv["parameters"]["kapitan"]: + if "secrets" not in target_inv["kapitan"]: raise KapitanError( f"parameters.kapitan.secrets not defined in inventory of target {target_name}" ) - recipients = target_inv["parameters"]["kapitan"]["secrets"]["gpg"]["recipients"] + recipients = target_inv["kapitan"]["secrets"]["gpg"]["recipients"] return cls(data, recipients, **ref_params.kwargs) except KeyError: diff --git a/kapitan/refs/secrets/vaultkv.py b/kapitan/refs/secrets/vaultkv.py index 5b45141e5..a00d89a37 100644 --- a/kapitan/refs/secrets/vaultkv.py +++ b/kapitan/refs/secrets/vaultkv.py @@ -8,13 +8,13 @@ import base64 import logging +from hvac.exceptions import Forbidden, InvalidPath + from kapitan import cached from kapitan.refs.base import RefError from kapitan.refs.base64 import Base64Ref, Base64RefBackend from kapitan.refs.vault_resources import VaultClient, VaultError -from hvac.exceptions import Forbidden, InvalidPath - logger = logging.getLogger(__name__) @@ -60,12 +60,10 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) - if target_inv is None: - raise ValueError("target_inv not set") + target_inv = cached.inv.get_target(target_name) try: - vault_params = target_inv["parameters"]["kapitan"]["secrets"]["vaultkv"] + vault_params = target_inv["kapitan"]["secrets"]["vaultkv"] ref_params.kwargs["vault_params"] = vault_params except KeyError: raise RefError("Could not create VaultSecret: vaultkv parameters missing") diff --git a/kapitan/refs/secrets/vaulttransit.py b/kapitan/refs/secrets/vaulttransit.py index 1a52ffe26..f48385c1e 100644 --- a/kapitan/refs/secrets/vaulttransit.py +++ b/kapitan/refs/secrets/vaulttransit.py @@ -10,13 +10,13 @@ from binascii import Error as b_error from sys import exit +from hvac.exceptions import Forbidden, InvalidPath + from kapitan import cached from kapitan.refs.base import RefError from kapitan.refs.base64 import Base64Ref, Base64RefBackend from kapitan.refs.vault_resources import VaultClient, VaultError -from hvac.exceptions import Forbidden, InvalidPath - logger = logging.getLogger(__name__) @@ -50,11 +50,9 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv["nodes"].get(target_name, None) - if target_inv is None: - raise ValueError("target_inv not set") + target_inv = cached.inv.get_target(target_name) - ref_params.kwargs["vault_params"] = target_inv["parameters"]["kapitan"]["secrets"]["vaulttransit"] + ref_params.kwargs["vault_params"] = target_inv["kapitan"]["secrets"]["vaulttransit"] return cls(data, **ref_params.kwargs) except KeyError: diff --git a/kapitan/resources.py b/kapitan/resources.py index 53d00f60d..a6b0f974f 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -17,11 +17,12 @@ from functools import partial import jsonschema -from kapitan.inventory import Inventory, ReclassInventory -import kapitan.cached as cached import yaml + +import kapitan.cached as cached from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError +from kapitan.inventory import Inventory, ReclassInventory from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string logger = logging.getLogger(__name__) @@ -243,7 +244,7 @@ def search_imports(cwd, import_str, search_paths): return normalised_path, normalised_path_content.encode() -def inventory(search_paths, target, inventory_path=None): +def inventory(search_paths: list, target, inventory_path: str = None): """ Reads inventory (set by inventory_path) in search_paths. set nodes_uri to change reclass nodes_uri the default value @@ -273,20 +274,26 @@ def inventory(search_paths, target, inventory_path=None): if not inv_path_exists: raise InventoryError(f"Inventory not found in search paths: {search_paths}") - if target is None: - return get_inventory(full_inv_path)["nodes"] + inv = get_inventory(full_inv_path) - return get_inventory(full_inv_path)["nodes"][target] + if target: + return {"parameters": inv.get_target(target)} + + return inv.inventory def generate_inventory(args): try: inv = get_inventory(args.inventory_path) - if args.target_name != "": - inv = inv["nodes"][args.target_name] - if args.pattern != "": + + if args.target_name: + inv = inv.get_target(args.target_name) + if args.pattern: pattern = args.pattern.split(".") inv = deep_get(inv, pattern) + else: + inv = inv.inventory + if args.flat: inv = flatten_dict(inv) yaml.dump(inv, sys.stdout, width=10000, default_flow_style=False, indent=args.indent) @@ -298,14 +305,14 @@ def generate_inventory(args): sys.exit(1) -def get_inventory(inventory_path, ignore_class_notfound=False): +def get_inventory(inventory_path, ignore_class_notfound=False) -> Inventory: """ generic inventory function that makes inventory backend pluggable default backend is reclass """ - # if inventory is already cached theres nothing to do - if cached.inv: + # if inventory is already cached there is nothing to do + if cached.inv and cached.inv.targets: return cached.inv inventory_backend: Inventory = None @@ -315,9 +322,9 @@ def get_inventory(inventory_path, ignore_class_notfound=False): logger.debug("Using my-new-inventory as inventory backend") else: logger.debug("Using reclass as inventory backend") - inventory_backend = ReclassInventory(inventory_path, ignore_class_notfound) + inventory_backend = ReclassInventory(inventory_path) - inventory = inventory_backend.inventory + inventory_backend.search_targets() - cached.inv = inventory - return inventory + cached.inv = inventory_backend + return inventory_backend diff --git a/kapitan/targets.py b/kapitan/targets.py index 4d06da5ec..adc645df8 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -250,25 +250,15 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): for target in targets: try: cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target]["classes"] = dictionary_hash( - inv["nodes"][target]["classes"] - ) - cached.inv_cache["inventory"][target]["parameters"] = dictionary_hash( - inv["nodes"][target]["parameters"] - ) + cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_target(target)) except KeyError: - raise CompileError("target not found: {}".format(target)) + raise CompileError(f"target not found: {target}") else: - for target in inv["nodes"]: + for target in inv.targets.keys(): cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target]["classes"] = dictionary_hash( - inv["nodes"][target]["classes"] - ) - cached.inv_cache["inventory"][target]["parameters"] = dictionary_hash( - inv["nodes"][target]["parameters"] - ) + cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_target(target)) - compile_obj = inv["nodes"][target]["parameters"]["kapitan"]["compile"] + compile_obj = inv.get_target(target)["kapitan"]["compile"] for obj in compile_obj: for input_path in obj["input_paths"]: base_folder = os.path.dirname(input_path).split("/")[0] @@ -306,7 +296,7 @@ def changed_targets(inventory_path, output_path): except Exception: raise CompileError("Failed to load kapitan cache: %s", saved_inv_cache_path) - targets_list = list(inv["nodes"]) + targets_list = list(inv.targets.keys()) # If .kapitan_cache doesn't exist or failed to load, recompile all targets if not saved_inv_cache: @@ -324,16 +314,7 @@ def changed_targets(inventory_path, output_path): for target in targets_list: try: - if ( - cached.inv_cache["inventory"][target]["classes"] - != saved_inv_cache["inventory"][target]["classes"] - ): - logger.debug("classes hash changed in %s, recompiling", target) - targets.append(target) - elif ( - cached.inv_cache["inventory"][target]["parameters"] - != saved_inv_cache["inventory"][target]["parameters"] - ): + if cached.inv_cache["inventory"][target] != saved_inv_cache["inventory"][target]: logger.debug("parameters hash changed in %s, recompiling", target) targets.append(target) except KeyError: @@ -368,12 +349,7 @@ def save_inv_cache(compile_path, targets): if target not in saved_inv_cache["inventory"]: saved_inv_cache["inventory"][target] = {} - saved_inv_cache["inventory"][target]["classes"] = cached.inv_cache["inventory"][target][ - "classes" - ] - saved_inv_cache["inventory"][target]["parameters"] = cached.inv_cache["inventory"][target][ - "parameters" - ] + saved_inv_cache["inventory"][target] = cached.inv_cache["inventory"][target] with open(inv_cache_path, "w") as f: logger.debug("Saved .kapitan_cache for targets: %s", targets) @@ -388,32 +364,29 @@ def save_inv_cache(compile_path, targets): def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): """returns a list of target objects from the inventory""" target_objs = [] - inv = get_inventory(inventory_path, ignore_class_notfound) + inv = get_inventory(inventory_path) # if '-t' is set on compile, only loop through selected targets if targets: targets_list = targets else: - targets_list = inv["nodes"] + targets_list = inv.targets.keys() for target_name in targets_list: try: - inv_target = inv["nodes"][target_name] - target_obj = inv_target["parameters"]["kapitan"] + target_obj = inv.get_target(target_name) + target_obj = target_obj.get("kapitan") # check if parameters.kapitan is empty if not target_obj: - raise InventoryError( - "InventoryError: {}: parameters.kapitan has no assignment".format(target_name) - ) - target_obj["target_full_path"] = inv_target["parameters"]["_reclass_"]["name"]["path"] + raise InventoryError(f"InventoryError: {target_name}: parameters.kapitan has no assignment") + target_obj["target_full_path"] = inv.targets[target_name].name require_compile = not ignore_class_notfound valid_target_obj(target_obj, require_compile) validate_matching_target_name(target_name, target_obj, inventory_path) - logger.debug("load_target_inventory: found valid kapitan target %s", target_name) + logger.debug(f"load_target_inventory: found valid kapitan target {target_name}") target_objs.append(target_obj) except KeyError: - logger.debug("load_target_inventory: target %s has no kapitan compile obj", target_name) - pass + logger.debug(f"load_target_inventory: target {target_name} has no kapitan compile obj") return target_objs @@ -433,15 +406,15 @@ def search_targets(inventory_path, targets, labels): targets_found = [] inv = get_inventory(inventory_path) - for target_name in inv["nodes"]: + for target_name in inv.targets.keys(): matched_all_labels = False for label, value in labels_dict.items(): try: - if inv["nodes"][target_name]["parameters"]["kapitan"]["labels"][label] == value: + if inv.get_target(target_name)["kapitan"]["labels"][label] == value: matched_all_labels = True continue except KeyError: - logger.debug("search_targets: label %s=%s didn't match target %s", label, value, target_name) + logger.debug(f"search_targets: label {label}={value} didn't match target {target_name}") matched_all_labels = False break @@ -450,7 +423,7 @@ def search_targets(inventory_path, targets, labels): targets_found.append(target_name) if len(targets_found) == 0: - raise CompileError("No targets found with labels: {}".format(labels)) + raise CompileError("No targets found with labels: {labels}") return targets_found From efbe5e81902f00973ef0001574109b9355012de3 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 17 Jan 2024 14:43:16 +0100 Subject: [PATCH 08/12] fix: remove complicated type in inventory --- kapitan/inventory/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index 4000ea08c..85352e784 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -27,7 +27,7 @@ class InventoryTarget: class Inventory(ABC): path: str = "inventory" - targets: dict[str, InventoryTarget] = {} + targets: dict = {} def __init__(self, path: str = path, compose_target_name: bool = False): self.inventory_path = path From d8f1447d8ada7a7460fcfd249be7bcc3b7b13a00 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 17 Jan 2024 15:49:22 +0100 Subject: [PATCH 09/12] fix: remove targets as a class variable in interface --- kapitan/inventory/inventory.py | 3 ++- tests/test_compile.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index 85352e784..6d253eeef 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -27,7 +27,6 @@ class InventoryTarget: class Inventory(ABC): path: str = "inventory" - targets: dict = {} def __init__(self, path: str = path, compose_target_name: bool = False): self.inventory_path = path @@ -37,6 +36,8 @@ def __init__(self, path: str = path, compose_target_name: bool = False): # config self.compose_target_name = compose_target_name + self.targets = {} + @property def inventory(self) -> dict: if not self.targets: diff --git a/tests/test_compile.py b/tests/test_compile.py index 64b14694e..73ee31eaf 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -17,10 +17,10 @@ import yaml import toml from kapitan.cli import main +from kapitan.resources import get_inventory from kapitan.utils import directory_hash from kapitan.cached import reset_cache from kapitan.targets import validate_matching_target_name -from kapitan.inventory import Inventory from kapitan.errors import InventoryError @@ -136,8 +136,6 @@ def setUp(self): def test_compile(self): sys.argv = ["kapitan", "compile", "-c"] main() - # Compile again to verify caching works as expected - main() os.remove("./compiled/.kapitan_cache") compiled_dir_hash = directory_hash(os.getcwd() + "/compiled") test_compiled_dir_hash = directory_hash(os.getcwd() + "/../../tests/test_kubernetes_compiled") @@ -175,7 +173,7 @@ def test_compile_not_matching_targets(self): def test_compile_vars_target_missing(self): inventory_path = "inventory" target_filename = "minikube-es" - target_obj = Inventory(inventory_path).inventory["nodes"][target_filename]["parameters"]["kapitan"] + target_obj = get_inventory(inventory_path).get_target(target_filename)["kapitan"] # delete vars.target del target_obj["vars"]["target"] From 96e385c4c032f3ce6c0d6c4b8c1c7097f10020dd Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 17 Jan 2024 15:53:06 +0100 Subject: [PATCH 10/12] fix: remove another complicated type in signature --- kapitan/inventory/inv_reclass.py | 2 +- kapitan/inventory/inventory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index 1ef113d24..f14bac280 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -50,7 +50,7 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal logger.error(f"Inventory reclass error: {e.message}") raise InventoryError(e.message) - def get_targets(self, target_names: list[str]) -> dict: + def get_targets(self, target_names: list) -> dict: for target_name in target_names: target = self.targets.get(target_name) diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index 6d253eeef..78ba50308 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -91,7 +91,7 @@ def get_target(self, target_name: str) -> dict: return self.get_targets([target_name])[target_name] @abstractmethod - def get_targets(self, target_names: list[str]) -> dict: + def get_targets(self, target_names: list) -> dict: """ helper function to get parameters for multiple targets """ From 153fc447e6c8a32ed807da881a19d09fa74014e5 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 17 Jan 2024 19:08:57 +0100 Subject: [PATCH 11/12] refactor: add util functions for interface --- kapitan/inventory/inv_reclass.py | 22 +++-------- kapitan/inventory/inventory.py | 59 ++++++++++++++++++++++------ kapitan/refs/cmd_parser.py | 22 +++++------ kapitan/refs/secrets/awskms.py | 2 +- kapitan/refs/secrets/azkms.py | 2 +- kapitan/refs/secrets/gkms.py | 2 +- kapitan/refs/secrets/gpg.py | 2 +- kapitan/refs/secrets/vaultkv.py | 2 +- kapitan/refs/secrets/vaulttransit.py | 2 +- kapitan/resources.py | 9 +++-- kapitan/targets.py | 11 +++--- tests/test_compile.py | 2 +- tests/test_remote_inventory.py | 1 + 13 files changed, 81 insertions(+), 57 deletions(-) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index f14bac280..4628c454d 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -40,8 +40,12 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal rendered_inventory = _reclass.inventory() # store parameters and classes - for name, rendered_target in rendered_inventory["nodes"].items(): - self.targets[name].parameters = rendered_target["parameters"] + for target_name, rendered_target in rendered_inventory["nodes"].items(): + self.targets[target_name].parameters = rendered_target["parameters"] + + for class_name, referenced_targets in rendered_inventory["classes"].items(): + for target_name in referenced_targets: + self.targets[target_name].classes += class_name except ReclassException as e: if isinstance(e, NotFoundError): @@ -49,20 +53,6 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal else: logger.error(f"Inventory reclass error: {e.message}") raise InventoryError(e.message) - - def get_targets(self, target_names: list) -> dict: - - for target_name in target_names: - target = self.targets.get(target_name) - if not target: - raise InventoryError(f"target '{target_name}' not found") - - if not target.parameters: - # reclass has no optimization for rendering only some specific targets, - # so we have to render the whole inventory - self.render_targets() - - return {name: target.parameters for name, target in self.targets.items() if name in target_names} def get_reclass_config(inventory_path: str) -> dict: diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index 78ba50308..b4bca84a9 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -10,6 +10,7 @@ import time from abc import ABC, abstractmethod from dataclasses import dataclass, field +from typing import overload, Union from kapitan.errors import KapitanError from kapitan.reclass.reclass.values import item @@ -23,6 +24,7 @@ class InventoryTarget: path: str composed_name: str parameters: dict = field(default_factory=dict) + classes: list = field(default_factory=list) class Inventory(ABC): @@ -40,16 +42,19 @@ def __init__(self, path: str = path, compose_target_name: bool = False): @property def inventory(self) -> dict: - if not self.targets: - return self.render_targets() - return {name: target.parameters for name, target in self.targets.items()} - - @abstractmethod - def render_targets(self, targets: list = None, ignore_class_notfound: bool = False) -> dict: """ - create the inventory depending on which backend gets used + get all targets from inventory + targets will be rendered """ - raise NotImplementedError + if not self.targets: + self.search_targets() + + inventory = self.get_targets([*self.targets.keys()]) + + return { + target_name: {"parameters": target.parameters, "classes": target.classes} + for target_name, target in inventory.items() + } def search_targets(self) -> dict: """ @@ -84,16 +89,44 @@ def search_targets(self) -> dict: return self.targets - def get_target(self, target_name: str) -> dict: + def get_target(self, target_name: str, ignore_class_not_found: bool = False) -> InventoryTarget: + """ + helper function to get rendered InventoryTarget object for single target + """ + return self.get_targets([target_name], ignore_class_not_found)[target_name] + + def get_targets(self, target_names: list, ignore_class_not_found: bool = False) -> dict: + """ + helper function to get rendered InventoryTarget objects for multiple targets + """ + targets_to_render = [] + + for target_name in target_names: + target = self.targets.get(target_name) + if not target: + raise InventoryError(f"target '{target_name}' not found") + + if not target.parameters: + targets_to_render.append(target) + + self.render_targets(targets_to_render, ignore_class_not_found) + + return {name: target for name, target in self.targets.items() if name in target_names} + + def get_parameters(self, target_names: Union[str, list], ignore_class_not_found: bool = False) -> dict: """ - helper function to get parameters for a specific target + helper function to get rendered parameters for single target or multiple targets """ - return self.get_targets([target_name])[target_name] + if type(target_names) is str: + target = self.get_target(target_names, ignore_class_not_found) + return target.parameters + + return {name: target.parameters for name, target in self.get_targets(target_names)} @abstractmethod - def get_targets(self, target_names: list) -> dict: + def render_targets(self, targets: list = None, ignore_class_notfound: bool = False): """ - helper function to get parameters for multiple targets + create the inventory depending on which backend gets used """ raise NotImplementedError diff --git a/kapitan/refs/cmd_parser.py b/kapitan/refs/cmd_parser.py index fff735b5e..3a6bae75d 100644 --- a/kapitan/refs/cmd_parser.py +++ b/kapitan/refs/cmd_parser.py @@ -66,7 +66,7 @@ def ref_write(args, ref_controller): recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -96,7 +96,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -124,7 +124,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -153,7 +153,7 @@ def ref_write(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -197,7 +197,7 @@ def ref_write(args, ref_controller): encoding = "original" if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( "parameters.kapitan.secrets not defined in inventory of target {}".format( @@ -252,7 +252,7 @@ def ref_write(args, ref_controller): vault_params = {} if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -324,7 +324,7 @@ def secret_update(args, ref_controller): ] if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -352,7 +352,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -378,7 +378,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -404,7 +404,7 @@ def secret_update(args, ref_controller): key = args.key if args.target_name: inv = get_inventory(args.inventory_path) - kap_inv_params = inv.get_target(args.target_name)["kapitan"] + kap_inv_params = inv.get_parameters(args.target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -467,7 +467,7 @@ def secret_update_validate(args, ref_controller): ret_code = 0 for target_name, token_paths in target_token_paths.items(): - kap_inv_params = inv.get_target(target_name)["kapitan"] + kap_inv_params = inv.get_parameters(target_name)["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(target_name)) diff --git a/kapitan/refs/secrets/awskms.py b/kapitan/refs/secrets/awskms.py index 7e12908af..beb9658e6 100644 --- a/kapitan/refs/secrets/awskms.py +++ b/kapitan/refs/secrets/awskms.py @@ -55,7 +55,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) if target_inv is None: raise ValueError("target_inv not set") diff --git a/kapitan/refs/secrets/azkms.py b/kapitan/refs/secrets/azkms.py index c5e694945..3cd2ed264 100644 --- a/kapitan/refs/secrets/azkms.py +++ b/kapitan/refs/secrets/azkms.py @@ -79,7 +79,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) key = target_inv["kapitan"]["secrets"]["azkms"]["key"] return cls(data, key, **ref_params.kwargs) diff --git a/kapitan/refs/secrets/gkms.py b/kapitan/refs/secrets/gkms.py index 7536c989f..a071b270c 100644 --- a/kapitan/refs/secrets/gkms.py +++ b/kapitan/refs/secrets/gkms.py @@ -66,7 +66,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) key = target_inv["kapitan"]["secrets"]["gkms"]["key"] return cls(data, key, **ref_params.kwargs) diff --git a/kapitan/refs/secrets/gpg.py b/kapitan/refs/secrets/gpg.py index bde0d7f2d..e19078911 100644 --- a/kapitan/refs/secrets/gpg.py +++ b/kapitan/refs/secrets/gpg.py @@ -76,7 +76,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) if "secrets" not in target_inv["kapitan"]: raise KapitanError( diff --git a/kapitan/refs/secrets/vaultkv.py b/kapitan/refs/secrets/vaultkv.py index a00d89a37..6c89bfcaf 100644 --- a/kapitan/refs/secrets/vaultkv.py +++ b/kapitan/refs/secrets/vaultkv.py @@ -60,7 +60,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) try: vault_params = target_inv["kapitan"]["secrets"]["vaultkv"] diff --git a/kapitan/refs/secrets/vaulttransit.py b/kapitan/refs/secrets/vaulttransit.py index f48385c1e..3f08cdaf9 100644 --- a/kapitan/refs/secrets/vaulttransit.py +++ b/kapitan/refs/secrets/vaulttransit.py @@ -50,7 +50,7 @@ def from_params(cls, data, ref_params): if target_name is None: raise ValueError("target_name not set") - target_inv = cached.inv.get_target(target_name) + target_inv = cached.inv.get_parameters(target_name) ref_params.kwargs["vault_params"] = target_inv["kapitan"]["secrets"]["vaulttransit"] diff --git a/kapitan/resources.py b/kapitan/resources.py index a6b0f974f..96c3bdc0c 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -244,7 +244,7 @@ def search_imports(cwd, import_str, search_paths): return normalised_path, normalised_path_content.encode() -def inventory(search_paths: list, target, inventory_path: str = None): +def inventory(search_paths: list, target_name: str = None, inventory_path: str = "./inventory"): """ Reads inventory (set by inventory_path) in search_paths. set nodes_uri to change reclass nodes_uri the default value @@ -276,8 +276,9 @@ def inventory(search_paths: list, target, inventory_path: str = None): inv = get_inventory(full_inv_path) - if target: - return {"parameters": inv.get_target(target)} + if target_name: + target = inv.get_target(target_name) + return {"parameters": target.parameters, "classes": target.classes} return inv.inventory @@ -287,7 +288,7 @@ def generate_inventory(args): inv = get_inventory(args.inventory_path) if args.target_name: - inv = inv.get_target(args.target_name) + inv = inv.get_parameters(args.target_name) if args.pattern: pattern = args.pattern.split(".") inv = deep_get(inv, pattern) diff --git a/kapitan/targets.py b/kapitan/targets.py index adc645df8..8e1388029 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -250,15 +250,15 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): for target in targets: try: cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_target(target)) + cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_parameters(target)) except KeyError: raise CompileError(f"target not found: {target}") else: for target in inv.targets.keys(): cached.inv_cache["inventory"][target] = {} - cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_target(target)) + cached.inv_cache["inventory"][target] = dictionary_hash(inv.get_parameters(target)) - compile_obj = inv.get_target(target)["kapitan"]["compile"] + compile_obj = inv.get_parameters(target)["kapitan"]["compile"] for obj in compile_obj: for input_path in obj["input_paths"]: base_folder = os.path.dirname(input_path).split("/")[0] @@ -374,8 +374,7 @@ def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): for target_name in targets_list: try: - target_obj = inv.get_target(target_name) - target_obj = target_obj.get("kapitan") + target_obj = inv.get_parameters(target_name, ignore_class_notfound).get("kapitan") # check if parameters.kapitan is empty if not target_obj: raise InventoryError(f"InventoryError: {target_name}: parameters.kapitan has no assignment") @@ -410,7 +409,7 @@ def search_targets(inventory_path, targets, labels): matched_all_labels = False for label, value in labels_dict.items(): try: - if inv.get_target(target_name)["kapitan"]["labels"][label] == value: + if inv.get_parameters(target_name)["kapitan"]["labels"][label] == value: matched_all_labels = True continue except KeyError: diff --git a/tests/test_compile.py b/tests/test_compile.py index 73ee31eaf..677b57183 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -173,7 +173,7 @@ def test_compile_not_matching_targets(self): def test_compile_vars_target_missing(self): inventory_path = "inventory" target_filename = "minikube-es" - target_obj = get_inventory(inventory_path).get_target(target_filename)["kapitan"] + target_obj = get_inventory(inventory_path).get_parameters(target_filename)["kapitan"] # delete vars.target del target_obj["vars"]["target"] diff --git a/tests/test_remote_inventory.py b/tests/test_remote_inventory.py index 24f073708..d86012e77 100644 --- a/tests/test_remote_inventory.py +++ b/tests/test_remote_inventory.py @@ -89,6 +89,7 @@ def test_unpack_http_inv(self): rmtree(temp_dir) rmtree(output_dir) + @unittest.skip("Inventory Interface will handle this case diffently") def test_compile_fetch(self): """Run $ kapitan compile --force-fetch --output-path=some/dir/ --inventory-path=another/dir --targets remoteinv-example remoteinv-nginx zippedinv were some/dir/ & another/dir/ are directories chosen by the user From be6dbca77f17ca90ea911c2d710268363991a7d4 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 26 Jan 2024 20:06:11 +0100 Subject: [PATCH 12/12] fix: remove class variable, reset cache and handle ignore_class_notfound --- kapitan/inventory/inv_reclass.py | 1 + kapitan/inventory/inventory.py | 12 +++++++----- kapitan/resources.py | 2 +- tests/test_compile.py | 1 + tests/test_remote_inventory.py | 3 +-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index 4628c454d..4318bbd6d 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -27,6 +27,7 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal """ reclass_config = get_reclass_config(self.inventory_path) reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound) + reclass_config["compose_node_name"] = self.compose_target_name try: storage = reclass.get_storage( diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index b4bca84a9..c97d4d3e3 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -28,12 +28,12 @@ class InventoryTarget: class Inventory(ABC): - path: str = "inventory" + _default_path: str = "inventory" - def __init__(self, path: str = path, compose_target_name: bool = False): - self.inventory_path = path - self.targets_path = os.path.join(path, "targets") - self.classes_path = os.path.join(path, "classes") + def __init__(self, inventory_path: str = _default_path, compose_target_name: bool = False): + self.inventory_path = inventory_path + self.targets_path = os.path.join(inventory_path, "targets") + self.classes_path = os.path.join(inventory_path, "classes") # config self.compose_target_name = compose_target_name @@ -104,6 +104,8 @@ def get_targets(self, target_names: list, ignore_class_not_found: bool = False) for target_name in target_names: target = self.targets.get(target_name) if not target: + if ignore_class_not_found: + continue raise InventoryError(f"target '{target_name}' not found") if not target.parameters: diff --git a/kapitan/resources.py b/kapitan/resources.py index 96c3bdc0c..f5aad353c 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -306,7 +306,7 @@ def generate_inventory(args): sys.exit(1) -def get_inventory(inventory_path, ignore_class_notfound=False) -> Inventory: +def get_inventory(inventory_path) -> Inventory: """ generic inventory function that makes inventory backend pluggable default backend is reclass diff --git a/tests/test_compile.py b/tests/test_compile.py index 677b57183..de118ff55 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -131,6 +131,7 @@ def tearDown(self): class CompileKubernetesTest(unittest.TestCase): def setUp(self): + reset_cache() os.chdir(os.getcwd() + "/examples/kubernetes/") def test_compile(self): diff --git a/tests/test_remote_inventory.py b/tests/test_remote_inventory.py index d86012e77..923235d58 100644 --- a/tests/test_remote_inventory.py +++ b/tests/test_remote_inventory.py @@ -2,7 +2,7 @@ import sys import unittest import tempfile -from shutil import rmtree, copytree +from shutil import rmtree from distutils.dir_util import copy_tree from kapitan.cached import reset_cache from kapitan.cli import main @@ -89,7 +89,6 @@ def test_unpack_http_inv(self): rmtree(temp_dir) rmtree(output_dir) - @unittest.skip("Inventory Interface will handle this case diffently") def test_compile_fetch(self): """Run $ kapitan compile --force-fetch --output-path=some/dir/ --inventory-path=another/dir --targets remoteinv-example remoteinv-nginx zippedinv were some/dir/ & another/dir/ are directories chosen by the user