add plugin to dump flattened config

This patch adds a new plugin 'dump' which resolves all kas config
includes and combines them in a single file. The format of the output
file can either be JSON or YAML. In case of YAML, the generated config
fulfills the kas-config schema and can be used as input to kas again.
In addition, the generated files can be used by external tools to
analyse the dependencies and versions of the project.

While the generated configuration is semantically identical to the
input config, we do not guarantee binary compatibility as especially
YAML provides multiple ways to serialize strings and null values.
Not giving this guarantee makes it easier to evolve and maintain the plugin.

The plugin itself extends the checkout plugin, as referenced repos have
to be checked-out first to resolve cross-repo references. This also
requires, that the declared refspec of all referenced repos can actually
be found and checked out.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

docs/userguide.rst

@@ -89,6 +89,11 @@ typically provides a single command.
 
 .. automodule:: kas.plugins.checkout
 
+``dump`` plugin
+~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: kas.plugins.dump
+
 ``for-all-repos`` plugin
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

kas/plugins/__init__.py

@@ -43,9 +43,11 @@ def load():
     from . import checkout
     from . import shell
     from . import menu
+    from . import dump
 
     register_plugins(build)
     register_plugins(checkout)
+    register_plugins(dump)
     register_plugins(for_all_repos)
     register_plugins(shell)
     register_plugins(menu)

kas/plugins/dump.py

@@ -0,0 +1,132 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017-2022
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+    This plugin implements the ``kas dump`` command.
+
+    When this command is executed, kas will parse all referenced config
+    files, expand includes and print a flattened yaml version of the
+    configuration to stdout. This config is semantically identical to the
+    input, but does not include any references to other configuration files.
+    The output of this command can be used to further analyse the build
+    configuration.
+
+    Please note:
+
+    - the dumped config is semantically identical but not bit-by-bit identical
+    - all referenced repositories are checked out to resolve cross-repo configs
+    - all refspecs are resolved before patches are applied
+
+    For example, to get a single config representing the final build config of
+    ``kas-project.yml:target-override.yml`` you could run:
+
+        kas dump kas-project.yml:target-override.yml > kas-project-expanded.yml
+
+    The generated config can be used as input for kas:
+
+        kas build kas-project-expanded.yml
+"""
+
+import logging
+import sys
+import json
+import yaml
+from collections import OrderedDict
+from kas.context import get_context
+from kas.plugins.checkout import Checkout
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2022'
+
+
+class Dump(Checkout):
+    """
+    Implements a kas plugin that combines multiple kas configurations
+    and dumps the result.
+    """
+
+    name = 'dump'
+    helpmsg = 'Expand and dump the final config.'
+
+    class KasYamlDumper(yaml.Dumper):
+        """
+        Yaml formatter (dumper) that generates output in a formatting which
+        is similar to kas example input files.
+        """
+
+        def represent_data(self, data):
+            if isinstance(data, str):
+                if data.count('\n') > 0:
+                    return self.represent_scalar(
+                        'tag:yaml.org,2002:str',
+                        data,
+                        style='|')
+                return self.represent_scalar('tag:yaml.org,2002:str', data)
+            elif isinstance(data, OrderedDict):
+                return self.represent_mapping(
+                    'tag:yaml.org,2002:map',
+                    data.items())
+            elif data is None:
+                return self.represent_scalar('tag:yaml.org,2002:null', '')
+            return super().represent_data(data)
+
+    @classmethod
+    def setup_parser(cls, parser):
+        super().setup_parser(parser)
+        parser.add_argument('--format',
+                            choices=['yaml', 'json'],
+                            default='yaml',
+                            help='Output format (default: yaml)')
+        parser.add_argument('--indent',
+                            type=int,
+                            default=4,
+                            help='Line indent (# of spaces, default: 4)')
+
+    def run(self, args):
+        args.skip += [
+            'setup_dir',
+            'repos_apply_patches',
+            'setup_environ',
+            'write_bbconfig',
+        ]
+
+        super().run(args)
+        ctx = get_context()
+        config_expanded = ctx.config.get_config()
+
+        # includes are already expanded, delete the key
+        if 'includes' in config_expanded['header']:
+            del config_expanded['header']['includes']
+
+        if args.format == 'json':
+            json.dump(config_expanded, sys.stdout, indent=args.indent)
+            sys.stdout.write('\n')
+        elif args.format == 'yaml':
+            yaml.dump(
+                config_expanded, sys.stdout,
+                indent=args.indent,
+                Dumper=self.KasYamlDumper)
+        else:
+            logging.error('invalid format %s', args.format)
+            sys.exit(1)
+
+
+__KAS_PLUGINS__ = [Dump]