Command line based configuration file merge

This extends the command line syntax for specifying configuration files.
You can now combine files by concatenating them, separated by colons:

kas build base.yml:board.yml:feature.yml

The motivation for this feature is to avoid having to write tons of
configuration files that perform this combinations statically via
includes.

In order to avoid complications and prevent that users shoot themselves
too easily into their feet, we deny the case of distributing the
configuration files over multiple repositories. Either all files
specified on the command line come from the same repo, or they are all
local (without versioning control).

Based on idea by Claudius Heine.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

docs/userguide.rst

@@ -246,6 +246,33 @@ different include files. Note that the order of the configuration file entries
 is not preserved within one include file, because the parser creates normal
 unordered dictionaries.
 
+Including configuration files via the command line
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When specifying the kas configuration file on the command line, additional
+configurations can be included ad-hoc:
+
+    $ kas build kas-base.yml:debug-image.yml:board.yml
+
+This is equivalent to static inclusion from some kas-combined.yml like this:
+
+.. code-block:: yaml
+
+    header:
+      version: x
+      includes:
+        - kas-base.yml
+        - debug.image.yml
+        - board.yml
+
+Command line inclusion allows to create configurations on-demand, without the
+need to write a kas configuration file for each possible combination.
+
+Note that all configuration files combined via the command line either have to
+come from the same repository or have to live outside of any versioning control.
+kas will refuse any other combination in order to avoid complications and
+configuration flaws that can easily emerge from them.
+
 Configuration reference
 ~~~~~~~~~~~~~~~~~~~~~~~
 

kas/config.py

@@ -25,6 +25,7 @@
 
 import os
 from .repos import Repo
+from .includehandler import IncludeHandler, IncludeException
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -35,10 +36,22 @@ class Config:
         Implements the kas configuration based on config files.
     """
     def __init__(self, filename, target, task=None):
-        from .includehandler import IncludeHandler
         self._config = {}
-        self.filename = os.path.abspath(filename)
-        self.handler = IncludeHandler(self.filename)
+        self.filenames = []
+
+        for configfile in filename.split(':'):
+            configfile = os.path.abspath(configfile)
+            repo_path = Repo.get_root_path(os.path.dirname(configfile),
+                                           fallback=False)
+            if self.filenames == []:
+                common_path = repo_path
+            elif repo_path != common_path:
+                raise IncludeException('All concatenated config files must '
+                                       'belong to the same repository or all '
+                                       'must be outside of versioning control')
+            self.filenames.append(configfile)
+
+        self.handler = IncludeHandler(self.filenames)
         self.repo_dict = self._get_repo_dict()
 
         if target:
@@ -75,7 +88,7 @@ class Config:
         """
         repo_config_dict = self._config.get('repos', {})
         repo_dict = {}
-        repo_fallback_path = os.path.dirname(self.filename)
+        repo_fallback_path = os.path.dirname(self.filenames[0])
         for repo in repo_config_dict:
             repo_config_dict[repo] = repo_config_dict[repo] or {}
             repo_dict[repo] = Repo.factory(repo,

kas/includehandler.py

@@ -120,8 +120,8 @@ class IncludeHandler:
         The includes are read and merged depth first from top to buttom.
     """
 
-    def __init__(self, top_file):
-        self.top_file = top_file
+    def __init__(self, top_files):
+        self.top_files = top_files
 
     def get_config(self, repos=None):
         """
@@ -253,7 +253,13 @@ class IncludeHandler:
                     dest[k] = upd[k]
             return dest
 
-        configs, missing_repos = _internal_include_handler(self.top_file)
+        configs = []
+        missing_repos = []
+        for configfile in self.top_files:
+            cfgs, reps = _internal_include_handler(configfile)
+            configs.extend(cfgs)
+            missing_repos.extend(reps)
+
         config = functools.reduce(_internal_dict_merge,
                                   map(lambda x: x[1], configs))
         return config, missing_repos

tests/test_includehandler.py

@@ -140,7 +140,7 @@ header:
         monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {})
         for test in testvector:
             with patch_open(includehandler, dictionary=test['fdict']):
-                ginc = includehandler.IncludeHandler('x.yml')
+                ginc = includehandler.IncludeHandler(['x.yml'])
                 config, missing = ginc.get_config(repos=test['rdict'])
 
                 # Remove header, because we dont want to compare it:
@@ -348,7 +348,7 @@ v: {v2: y, v3: y, v5: y}'''),
                 os.path.abspath('z.yml'): header.format('''
 v: {v3: z, v4: z}''')}
         with patch_open(includehandler, dictionary=data):
-            ginc = includehandler.IncludeHandler('x.yml')
+            ginc = includehandler.IncludeHandler(['x.yml'])
             config, _ = ginc.get_config()
             keys = list(config['v'].keys())
             index = {keys[i]: i for i in range(len(keys))}