From 2077900b4e2049d515991cc47400395ac88e9998 Mon Sep 17 00:00:00 2001 From: Jan Kiszka Date: Wed, 18 Aug 2021 15:40:29 +0200 Subject: [PATCH] includehandler: Fix repo path detection for local includes So far, repository paths for local includes were derived from the path name of the config file containing the include, rather than using the actual repository root as specified in the documentation. No one complained so far, some layers simply adjusted their includes to this inconsistency which was only discovered during refactorings. Fix this issue by passing also the repository path along with the config filename down the recursive _internal_include_handler calls. The top-level repo path now needs to be retrieved before the creation of IncludeHandler and passed to it then. This has the side effect of enabling deviating top-level repo paths, a feature that will be used by the upcoming changes for a .config.yaml file in KAS_WORK_DIR. As there are existing users of the old behavior out there, fall back to it if a local include cannot be found under the correct path and warn if this succeeds. This allows smooth migration of layers to the right pattern as they update their kas version. Signed-off-by: Jan Kiszka --- kas/config.py | 11 ++++++----- kas/includehandler.py | 38 ++++++++++++++++++++++++------------ tests/test_includehandler.py | 4 ++-- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/kas/config.py b/kas/config.py index 0119f68..e7ae124 100644 --- a/kas/config.py +++ b/kas/config.py @@ -1,6 +1,6 @@ # kas - setup tool for bitbake based projects # -# Copyright (c) Siemens AG, 2017-2020 +# Copyright (c) Siemens AG, 2017-2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,7 @@ from .repos import Repo from .includehandler import IncludeHandler, IncludeException __license__ = 'MIT' -__copyright__ = 'Copyright (c) Siemens AG, 2017-2018' +__copyright__ = 'Copyright (c) Siemens AG, 2017-2021' class Config: @@ -41,6 +41,8 @@ class Config: self._config = {} self.filenames = [os.path.abspath(configfile) for configfile in filename.split(':')] + self.top_repo_path = Repo.get_root_path( + os.path.dirname(self.filenames[0])) repo_paths = [Repo.get_root_path(os.path.dirname(configfile), fallback=False) @@ -50,7 +52,7 @@ class Config: 'belong to the same repository or all ' 'must be outside of versioning control') - self.handler = IncludeHandler(self.filenames) + self.handler = IncludeHandler(self.filenames, self.top_repo_path) self.repo_dict = self._get_repo_dict() def get_build_system(self): @@ -88,13 +90,12 @@ class Config: repo_config_dict = self._config.get('repos', {}) repo_defaults = self._config.get('defaults', {}).get('repos', {}) repo_dict = {} - repo_fallback_path = os.path.dirname(self.filenames[0]) for repo in repo_config_dict: repo_config_dict[repo] = repo_config_dict[repo] or {} repo_dict[repo] = Repo.factory(repo, repo_config_dict[repo], repo_defaults, - repo_fallback_path) + self.top_repo_path) return repo_dict diff --git a/kas/includehandler.py b/kas/includehandler.py index 5964a32..c8f76a5 100644 --- a/kas/includehandler.py +++ b/kas/includehandler.py @@ -1,6 +1,6 @@ # kas - setup tool for bitbake based projects # -# Copyright (c) Siemens AG, 2017-2018 +# Copyright (c) Siemens AG, 2017-2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -37,7 +37,7 @@ from . import __file_version__, __compatible_file_version__ from . import CONFIGSCHEMA __license__ = 'MIT' -__copyright__ = 'Copyright (c) Siemens AG, 2017-2018' +__copyright__ = 'Copyright (c) Siemens AG, 2017-2021' class LoadConfigException(Exception): @@ -126,8 +126,9 @@ class IncludeHandler: The includes are read and merged from the deepest level upwards. """ - def __init__(self, top_files): + def __init__(self, top_files, top_repo_path): self.top_files = top_files + self.top_repo_path = top_repo_path def get_config(self, repos=None): """ @@ -143,7 +144,7 @@ class IncludeHandler: repos = repos or {} - def _internal_include_handler(filename): + def _internal_include_handler(filename, repo_path): """ Recursively loads include files and finds missing repos. @@ -185,10 +186,21 @@ class IncludeHandler: includefile = include else: includefile = os.path.abspath( - os.path.join( - os.path.dirname(filename), - include)) - (cfg, rep) = _internal_include_handler(includefile) + os.path.join(repo_path, include)) + if not os.path.exists(includefile): + alternate = os.path.abspath( + os.path.join(os.path.dirname(filename), + include)) + if os.path.exists(alternate): + logging.warning( + 'Falling back to file-relative addressing ' + 'of local include "%s"', include) + logging.warning( + 'Update your layer to repo-relative ' + 'addressing to avoid this warning') + includefile = alternate + (cfg, rep) = _internal_include_handler(includefile, + repo_path) configs.extend(cfg) missing_repos.extend(rep) elif isinstance(include, Mapping): @@ -206,11 +218,10 @@ class IncludeHandler: raise IncludeException( '"file" is not specified: {}' .format(include)) + abs_includedir = os.path.abspath(includedir) (cfg, rep) = _internal_include_handler( - os.path.abspath( - os.path.join( - includedir, - includefile))) + os.path.join(abs_includedir, includefile), + abs_includedir) configs.extend(cfg) missing_repos.extend(rep) else: @@ -261,7 +272,8 @@ class IncludeHandler: configs = [] missing_repos = [] for configfile in self.top_files: - cfgs, reps = _internal_include_handler(configfile) + cfgs, reps = _internal_include_handler(configfile, + self.top_repo_path) configs.extend(cfgs) for repo in reps: if repo not in missing_repos: diff --git a/tests/test_includehandler.py b/tests/test_includehandler.py index 6ca959a..ac7e613 100644 --- a/tests/test_includehandler.py +++ b/tests/test_includehandler.py @@ -139,7 +139,7 @@ header: monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {}) for test in testvector: with patch_open(includehandler, dictionary=test['fdict']): - ginc = includehandler.IncludeHandler(['x.yml']) + ginc = includehandler.IncludeHandler(['x.yml'], '.') config, missing = ginc.get_config(repos=test['rdict']) # Remove header, because we dont want to compare it: @@ -347,7 +347,7 @@ v: {v2: y, v3: y, v5: y}'''), os.path.abspath('z.yml'): header.format(''' v: {v3: z, v4: z}''')} with patch_open(includehandler, dictionary=data): - ginc = includehandler.IncludeHandler(['x.yml']) + ginc = includehandler.IncludeHandler(['x.yml'], '.') config, _ = ginc.get_config() keys = list(config['v'].keys()) index = {keys[i]: i for i in range(len(keys))}