wip

kapitan/cli.py

@@ -25,7 +25,7 @@
 from kapitan.refs.base import RefController, Revealer
 from kapitan.refs.cmd_parser import handle_refs_command
 from kapitan.resources import generate_inventory, resource_callbacks, search_imports
-from kapitan.targets import compile_targets, schema_validate_compiled
+from kapitan.targets import compile_targets
 from kapitan.utils import check_version, from_dot_kapitan, searchvar
 from kapitan.version import DESCRIPTION, PROJECT_NAME, VERSION
 
@@ -584,7 +584,7 @@ def build_parser():
         aliases=["v"],
         help="validates the compile output against schemas as specified in inventory",
     )
-    validate_parser.set_defaults(func=schema_validate_compiled, name="validate")
+    # validate_parser.set_defaults(func=schema_validate_compiled, name="validate")
 
     validate_parser.add_argument(
         "--compiled-path",

kapitan/inputs/base.py

@@ -18,10 +18,25 @@
 from kapitan.errors import CompileError, KapitanError
 from kapitan.refs.base import Revealer
 from kapitan.utils import PrettyDumper
+from enum import Enum
 
 logger = logging.getLogger(__name__)
 
 
+class InputTypes(Enum):
+    """
+    All currently supported input types in the inventory for parameters.kapitan.compile
+    """
+
+    KADET = "kadet"
+    JSONNET = "jsonnet"
+    JINJA2 = "jinja2"
+    HELM = "helm"
+    EXTERNAL = "external"
+    COPY = "copy"
+    REMOVE = "remove"
+
+
 class InputType(object):
     def __init__(self, type_name, compile_path, search_paths, ref_controller):
         self.type_name = type_name

kapitan/inventory/__init__.py

@@ -1 +1,2 @@
 from .base import Inventory
+from .reclass import ReclassInventory

kapitan/inventory/base.py

@@ -6,56 +6,179 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
+import logging
+import os
+from kapitan import cached
 
-from kapitan.errors import InventoryError
+from kapitan.errors import InventoryError, KapitanError
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class InventoryTarget(dict):
+    targets_path: str
+
+    def __init__(self, name: str, path: str):
+        self.name = name
+        self.path = path
+
+        # compose node name
+        self.composed_name = os.path.splitext(path)[0].replace(path + os.sep, "").replace("/", ".")
+
+    @property
+    def kapitan(self) -> dict():
+        kapitan_spec = self.get("kapitan")
+        if not kapitan_spec:
+            raise InventoryValidationError("parameters.kapitan is empty")
+
+        return kapitan_spec
+
+    @property
+    def parameters(self) -> dict():
+        parameters = self.get("parameters")
+        if not parameters:
+            raise InventoryValidationError("parameters is empty")
+
+        return parameters
 
 
 class Inventory(ABC):
-    _instance: object
+    _instance = None
 
     def __new__(cls, *args, **kwargs):
         """
         create only one specific inventory instance (singleton)
         """
-        if not cls._instance:
-            # initialize new instance
-            cls._instance = super(Inventory, cls).__new__(cls)
-            cls._instance.__init__(**args, **kwargs)
-        return cls._instance
+        if Inventory._instance is None:
+            Inventory._instance = super().__new__(cls)
+        return Inventory._instance
 
     def __init__(self, inventory_path: str, compose_node_name: dict = False):
         self.inventory_path = inventory_path
+        self.targets_path = os.path.join(inventory_path, "targets")
+        self.classes_path = os.path.join(inventory_path, "classes")
+
+        # config
         self.compose_node_name = compose_node_name
-        self._parameters = None
+
+        # used as cache for inventory
+        self._targets = {}
+        self._parameters = {}
+
+    @classmethod
+    def get(cls):
+        if cls._instance is None:
+            raise InventoryError("no type specified")
+        return cls._instance
 
     @property
     def inventory(self) -> dict:
         if not self._parameters:
             self._parameters = self.get_parameters()
+            cached.inv = self._parameters
         return self._parameters
 
     @abstractmethod
-    def get_parameters(self, ignore_class_notfound=False) -> dict:
+    def get_parameters(self, ignore_class_notfound: bool = False) -> dict:
         """
         create the inventory depending which backend gets used
         """
         raise NotImplementedError
 
-    def search_targets(self) -> list:
+    def search_targets(self) -> dict:
         """
-        look for targets at '<inventory_path>/targets/'
+        look for targets at '<inventory_path>/targets/' and return targets
         """
-        targets = []
-        return targets
+        for root, _, files in os.walk(self.targets_path):
+            for file in files:
+                # split file extension and check if yml/yaml
+                path = os.path.join(root, file)
+                name, ext = os.path.splitext(file)
+                if ext not in (".yml", ".yaml"):
+                    logger.debug(f"{file}: targets have to be .yml or .yaml files.")
+                    continue
+
+                # initialize target
+                target = InventoryTarget(name, path)
+                if self.compose_node_name:
+                    target.name = target.composed_name
+
+                # check for same name
+                if self._targets.get(target.name):
+                    raise InventoryError(
+                        f"Conflicting targets {target.name}: {target.path} and {self._targets[target.name].path}"
+                    )
+
+                self._targets[target.name] = target
+
+        return self._targets
 
     def get_target(self, target_name: str) -> dict:
         """
         get parameters for a specific target
         """
-        target = self.inventory.get(target_name)
-        if not target:
-            raise InventoryError(f"target '{target_name}' not found")
-        return target
+        return self.get_targets([target_name])
 
-    def fetch_dependencies(self):
+    @abstractmethod
+    def get_targets(self, target_names: list[str]) -> dict:
+        """
+        get parameters for multiple targets
+        """
         raise NotImplementedError
+
+    def fetch_dependencies(self, fetch, force_fetch):
+        # fetch inventory
+        if fetch:
+            # new_source checks for new sources in fetched inventory items
+            new_sources = list(set(list_sources(target_objs)) - cached.inv_sources)
+            while new_sources:
+                fetch_inventories(
+                    inventory_path,
+                    target_objs,
+                    dep_cache_dir,
+                    force_fetch,
+                    pool,
+                )
+                cached.reset_inv()
+                target_objs = load_target_inventory(updated_targets)
+                cached.inv_sources.update(new_sources)
+                new_sources = list(set(list_sources(target_objs)) - cached.inv_sources)
+            # reset inventory cache and load target objs to check for missing classes
+            cached.reset_inv()
+            target_objs = load_target_inventory(updated_targets)
+        # fetch dependencies
+        if fetch:
+            fetch_dependencies(output_path, target_objs, dep_cache_dir, force_fetch, pool)
+        # fetch targets which have force_fetch: true
+        elif not kwargs.get("force_fetch", False):
+            fetch_objs = []
+            # iterate through targets
+            for target in target_objs:
+                try:
+                    # get value of "force_fetch" property
+                    dependencies = target["dependencies"]
+                    # dependencies is still a list
+                    for entry in dependencies:
+                        force_fetch = entry["force_fetch"]
+                        if force_fetch:
+                            fetch_objs.append(target)
+                except KeyError:
+                    # targets may have no "dependencies" or "force_fetch" key
+                    continue
+            # fetch dependencies from targets with force_fetch set to true
+            if fetch_objs:
+                fetch_dependencies(output_path, fetch_objs, dep_cache_dir, True, pool)
+
+
+class InventoryError(KapitanError):
+    """inventory error"""
+
+    pass
+
+
+class InventoryValidationError(InventoryError):
+    """inventory validation error"""
+
+    pass

kapitan/inventory/reclass.py

@@ -13,7 +13,7 @@
 
 class ReclassInventory(Inventory):
 
-    def get_parameters(self, ignore_class_notfound=False) -> dict:
+    def get_parameters(self, ignore_class_notfound: bool = False) -> dict:
         """
         Runs a reclass inventory in inventory_path
         (same output as running ./reclass.py -b inv_base_uri/ --inventory)
@@ -36,13 +36,25 @@ def get_parameters(self, ignore_class_notfound=False) -> dict:
             class_mappings = reclass_config.get("class_mappings")  # this defaults to None (disabled)
             _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config))
 
-            return _reclass.inventory()
+            return {name: node["parameters"] for name, node in _reclass.inventory()["nodes"].items()}
         except ReclassException as e:
             if isinstance(e, NotFoundError):
                 logger.error("Inventory reclass error: inventory not found")
             else:
                 logger.error(f"Inventory reclass error: {e.message}")
             raise InventoryError(e.message)
+
+    def get_targets(self, target_names: list[str]) -> dict:
+        targets = {}
+
+        for target_name in target_names:
+            target = self.inventory.get(target_name)
+            if not target:
+                raise InventoryError(f"target '{target_name}' not found")
+
+            targets[target_name] = target
+
+        return targets
 
 
     def get_config(self) -> dict:
@@ -65,13 +77,13 @@ def get_config(self) -> dict:
         if os.path.isfile(cfg_file):
             with open(cfg_file, "r") as fp:
                 config = yaml.load(fp.read(), Loader=YamlLoader)
-                logger.debug("Using reclass inventory config at: {}".format(cfg_file))
+                logger.debug(f"Using reclass inventory config at: {cfg_file}")
             if config:
                 # set attributes, take default values if not present
                 for key, value in config.items():
                     reclass_config[key] = value
             else:
-                logger.debug("{}: Empty config file. Using reclass inventory config defaults".format(cfg_file))
+                logger.debug(f"Reclass config: Empty config file at {cfg_file}. Using reclass inventory config defaults")
         else:
             logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults")