From 17691d27566ceb4e39691a907e66faca991fbc64 Mon Sep 17 00:00:00 2001 From: Claudius Heine Date: Thu, 22 Jun 2017 12:28:32 +0200 Subject: [PATCH] Refactored fetching and checking out of missing repos in kas.config The current implementation did some more hackish solution to work around some of kas infrastructure, without changing to much. This patch cleans this up and therefor remove the more obsure part of the include mechansim. Signed-off-by: Claudius Heine --- kas/config.py | 40 +++------------------------- kas/libcmds.py | 64 +++----------------------------------------- kas/libkas.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 97 deletions(-) diff --git a/kas/config.py b/kas/config.py index a48f924..85659b6 100644 --- a/kas/config.py +++ b/kas/config.py @@ -43,7 +43,7 @@ except ImportError: return platform.dist()[0] from .repos import Repo -from .libkas import run_cmd +from .libkas import run_cmd, repo_fetch, repo_checkout __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017' @@ -329,8 +329,10 @@ class ConfigStatic(Config): missing_repos_old = missing_repos if missing_repos: complete = False - self._fetch_missing_repos(missing_repos) repo_dict = self.get_repo_dict() + for repo in missing_repos: + repo_fetch(self, repo_dict[repo]) + repo_checkout(self, repo_dict[repo]) repos = {r: repo_dict[r].path for r in repo_dict} def get_repos(self): @@ -387,40 +389,6 @@ class ConfigStatic(Config): repo_dict[repo] = rep return repo_dict - def _fetch_missing_repos(self, missing_repos): - """ - Fetches all repos from the missing_repos list. - """ - from .libcmds import (Macro, ReposFetch, ReposCheckout) - - class MissingRepoConfig(Config): - """ - Custom config class, because we only want to - fetch the missing repositories needed for a - complete configuration not all of them. - """ - def __init__(self, outerself): - super().__init__() - self.outerself = outerself - self.filename = outerself.filename - self.environ = outerself.environ - self.__kas_work_dir = outerself.kas_work_dir - - def get_repo_ref_dir(self): - return self.outerself.get_repo_ref_dir() - - def get_proxy_config(self): - return self.outerself.get_proxy_config() - - def get_repos(self): - return list(map(lambda x: self.outerself.get_repo_dict()[x], - missing_repos)) - - macro = Macro() - macro.add(ReposFetch()) - macro.add(ReposCheckout()) - macro.run(MissingRepoConfig(self)) - def load_config(filename, target): """ diff --git a/kas/libcmds.py b/kas/libcmds.py index 91133f0..3b13bd3 100644 --- a/kas/libcmds.py +++ b/kas/libcmds.py @@ -28,7 +28,7 @@ import logging import shutil import os from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check, - run_cmd, get_build_environ) + get_build_environ, repo_fetch, repo_checkout) __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017' @@ -210,42 +210,7 @@ class ReposFetch(Command): def execute(self, config): for repo in config.get_repos(): - if repo.git_operation_disabled: - continue - - if not os.path.exists(repo.path): - os.makedirs(os.path.dirname(repo.path), exist_ok=True) - gitsrcdir = os.path.join(config.get_repo_ref_dir() or '', - repo.qualified_name) - logging.debug('Looking for repo ref dir in %s', gitsrcdir) - if config.get_repo_ref_dir() and os.path.exists(gitsrcdir): - run_cmd(['/usr/bin/git', - 'clone', - '--reference', gitsrcdir, - repo.url, repo.path], - env=config.environ, - cwd=config.kas_work_dir) - else: - run_cmd(['/usr/bin/git', 'clone', '-q', repo.url, - repo.path], - env=config.environ, - cwd=config.kas_work_dir) - continue - - # Does refspec in the current repository? - (retc, output) = run_cmd(['/usr/bin/git', 'cat-file', - '-t', repo.refspec], env=config.environ, - cwd=repo.path, fail=False) - if retc == 0: - continue - - # No it is missing, try to fetch - (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'], - env=config.environ, - cwd=repo.path, fail=False) - if retc: - logging.warning('Could not update repository %s: %s', - repo.name, output) + repo_fetch(config, repo) class ReposCheckout(Command): @@ -258,27 +223,4 @@ class ReposCheckout(Command): def execute(self, config): for repo in config.get_repos(): - if repo.git_operation_disabled: - continue - - # Check if repos is dirty - (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'], - env=config.environ, cwd=repo.path, - fail=False) - if len(output): - logging.warning('Repo %s is dirty. no checkout', repo.name) - continue - - # Check if current HEAD is what in the config file is defined. - (_, output) = run_cmd(['/usr/bin/git', 'rev-parse', - '--verify', 'HEAD'], - env=config.environ, cwd=repo.path) - - if output.strip() == repo.refspec: - logging.info('Repo %s has already checkout out correct ' - 'refspec. nothing to do', repo.name) - continue - - run_cmd(['/usr/bin/git', 'checkout', '-q', - '{refspec}'.format(refspec=repo.refspec)], - cwd=repo.path) + repo_checkout(config, repo) diff --git a/kas/libkas.py b/kas/libkas.py index 3c08acc..5e8b147 100644 --- a/kas/libkas.py +++ b/kas/libkas.py @@ -158,6 +158,78 @@ def find_program(paths, name): return None +def repo_fetch(config, repo): + """ + Fetches the repository to the kas_work_dir. + """ + if repo.git_operation_disabled: + return + + if not os.path.exists(repo.path): + os.makedirs(os.path.dirname(repo.path), exist_ok=True) + gitsrcdir = os.path.join(config.get_repo_ref_dir() or '', + repo.qualified_name) + logging.debug('Looking for repo ref dir in %s', gitsrcdir) + if config.get_repo_ref_dir() and os.path.exists(gitsrcdir): + run_cmd(['/usr/bin/git', + 'clone', + '--reference', gitsrcdir, + repo.url, repo.path], + env=config.environ, + cwd=config.kas_work_dir) + else: + run_cmd(['/usr/bin/git', 'clone', '-q', repo.url, + repo.path], + env=config.environ, + cwd=config.kas_work_dir) + return + + # Does refspec in the current repository? + (retc, output) = run_cmd(['/usr/bin/git', 'cat-file', + '-t', repo.refspec], env=config.environ, + cwd=repo.path, fail=False) + if retc == 0: + return + + # No it is missing, try to fetch + (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'], + env=config.environ, + cwd=repo.path, fail=False) + if retc: + logging.warning('Could not update repository %s: %s', + repo.name, output) + + +def repo_checkout(config, repo): + """ + Checks out the correct revision of the repo. + """ + if repo.git_operation_disabled: + return + + # Check if repos is dirty + (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'], + env=config.environ, cwd=repo.path, + fail=False) + if len(output): + logging.warning('Repo %s is dirty. no checkout', repo.name) + return + + # Check if current HEAD is what in the config file is defined. + (_, output) = run_cmd(['/usr/bin/git', 'rev-parse', + '--verify', 'HEAD'], + env=config.environ, cwd=repo.path) + + if output.strip() == repo.refspec: + logging.info('Repo %s has already checkout out correct ' + 'refspec. nothing to do', repo.name) + return + + run_cmd(['/usr/bin/git', 'checkout', '-q', + '{refspec}'.format(refspec=repo.refspec)], + cwd=repo.path) + + def get_build_environ(config, build_dir): """ Create the build environment variables.