includehandler: Fix repo path detection for local includes

So far, repository paths for local includes were derived from the path
name of the config file containing the include, rather than using the
actual repository root as specified in the documentation. No one
complained so far, some layers simply adjusted their includes to this
inconsistency which was only discovered during refactorings.

Fix this issue by passing also the repository path along with the config
filename down the recursive _internal_include_handler calls. The
top-level repo path now needs to be retrieved before the creation of
IncludeHandler and passed to it then. This has the side effect of
enabling deviating top-level repo paths, a feature that will be used by
the upcoming changes for a .config.yaml file in KAS_WORK_DIR.

As there are existing users of the old behavior out there, fall back to
it if a local include cannot be found under the correct path and warn if
this succeeds. This allows smooth migration of layers to the right
pattern as they update their kas version.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

kas/config.py

@@ -1,6 +1,6 @@
 # kas - setup tool for bitbake based projects
 #
-# Copyright (c) Siemens AG, 2017-2020
+# Copyright (c) Siemens AG, 2017-2021
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -28,7 +28,7 @@ from .repos import Repo
 from .includehandler import IncludeHandler, IncludeException
 
 __license__ = 'MIT'
-__copyright__ = 'Copyright (c) Siemens AG, 2017-2018'
+__copyright__ = 'Copyright (c) Siemens AG, 2017-2021'
 
 
 class Config:
@@ -41,6 +41,8 @@ class Config:
         self._config = {}
         self.filenames = [os.path.abspath(configfile)
                           for configfile in filename.split(':')]
+        self.top_repo_path = Repo.get_root_path(
+            os.path.dirname(self.filenames[0]))
 
         repo_paths = [Repo.get_root_path(os.path.dirname(configfile),
                                          fallback=False)
@@ -50,7 +52,7 @@ class Config:
                                    'belong to the same repository or all '
                                    'must be outside of versioning control')
 
-        self.handler = IncludeHandler(self.filenames)
+        self.handler = IncludeHandler(self.filenames, self.top_repo_path)
         self.repo_dict = self._get_repo_dict()
 
     def get_build_system(self):
@@ -88,13 +90,12 @@ class Config:
         repo_config_dict = self._config.get('repos', {})
         repo_defaults = self._config.get('defaults', {}).get('repos', {})
         repo_dict = {}
-        repo_fallback_path = os.path.dirname(self.filenames[0])
         for repo in repo_config_dict:
             repo_config_dict[repo] = repo_config_dict[repo] or {}
             repo_dict[repo] = Repo.factory(repo,
                                            repo_config_dict[repo],
                                            repo_defaults,
-                                           repo_fallback_path)
+                                           self.top_repo_path)
 
         return repo_dict
 

kas/includehandler.py

@@ -1,6 +1,6 @@
 # kas - setup tool for bitbake based projects
 #
-# Copyright (c) Siemens AG, 2017-2018
+# Copyright (c) Siemens AG, 2017-2021
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -37,7 +37,7 @@ from . import __file_version__, __compatible_file_version__
 from . import CONFIGSCHEMA
 
 __license__ = 'MIT'
-__copyright__ = 'Copyright (c) Siemens AG, 2017-2018'
+__copyright__ = 'Copyright (c) Siemens AG, 2017-2021'
 
 
 class LoadConfigException(Exception):
@@ -126,8 +126,9 @@ class IncludeHandler:
         The includes are read and merged from the deepest level upwards.
     """
 
-    def __init__(self, top_files):
+    def __init__(self, top_files, top_repo_path):
         self.top_files = top_files
+        self.top_repo_path = top_repo_path
 
     def get_config(self, repos=None):
         """
@@ -143,7 +144,7 @@ class IncludeHandler:
 
         repos = repos or {}
 
-        def _internal_include_handler(filename):
+        def _internal_include_handler(filename, repo_path):
             """
             Recursively loads include files and finds missing repos.
 
@@ -185,10 +186,21 @@ class IncludeHandler:
                         includefile = include
                     else:
                         includefile = os.path.abspath(
-                            os.path.join(
-                                os.path.dirname(filename),
-                                include))
-                    (cfg, rep) = _internal_include_handler(includefile)
+                            os.path.join(repo_path, include))
+                        if not os.path.exists(includefile):
+                            alternate = os.path.abspath(
+                                os.path.join(os.path.dirname(filename),
+                                             include))
+                            if os.path.exists(alternate):
+                                logging.warning(
+                                    'Falling back to file-relative addressing '
+                                    'of local include "%s"', include)
+                                logging.warning(
+                                    'Update your layer to repo-relative '
+                                    'addressing to avoid this warning')
+                                includefile = alternate
+                    (cfg, rep) = _internal_include_handler(includefile,
+                                                           repo_path)
                     configs.extend(cfg)
                     missing_repos.extend(rep)
                 elif isinstance(include, Mapping):
@@ -206,11 +218,10 @@ class IncludeHandler:
                             raise IncludeException(
                                 '"file" is not specified: {}'
                                 .format(include))
+                        abs_includedir = os.path.abspath(includedir)
                         (cfg, rep) = _internal_include_handler(
-                            os.path.abspath(
-                                os.path.join(
-                                    includedir,
-                                    includefile)))
+                            os.path.join(abs_includedir, includefile),
+                            abs_includedir)
                         configs.extend(cfg)
                         missing_repos.extend(rep)
                     else:
@@ -261,7 +272,8 @@ class IncludeHandler:
         configs = []
         missing_repos = []
         for configfile in self.top_files:
-            cfgs, reps = _internal_include_handler(configfile)
+            cfgs, reps = _internal_include_handler(configfile,
+                                                   self.top_repo_path)
             configs.extend(cfgs)
             for repo in reps:
                 if repo not in missing_repos:

tests/test_includehandler.py

@@ -139,7 +139,7 @@ header:
         monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {})
         for test in testvector:
             with patch_open(includehandler, dictionary=test['fdict']):
-                ginc = includehandler.IncludeHandler(['x.yml'])
+                ginc = includehandler.IncludeHandler(['x.yml'], '.')
                 config, missing = ginc.get_config(repos=test['rdict'])
 
                 # Remove header, because we dont want to compare it:
@@ -347,7 +347,7 @@ v: {v2: y, v3: y, v5: y}'''),
                 os.path.abspath('z.yml'): header.format('''
 v: {v3: z, v4: z}''')}
         with patch_open(includehandler, dictionary=data):
-            ginc = includehandler.IncludeHandler(['x.yml'])
+            ginc = includehandler.IncludeHandler(['x.yml'], '.')
             config, _ = ginc.get_config()
             keys = list(config['v'].keys())
             index = {keys[i]: i for i in range(len(keys))}