Refactored fetching and checking out of missing repos in kas.config

The current implementation did some more hackish solution to work
around some of kas infrastructure, without changing to much.

This patch cleans this up and therefor remove the more obsure part
of the include mechansim.

Signed-off-by: Claudius Heine <ch@denx.de>

kas/config.py

@@ -43,7 +43,7 @@ except ImportError:
         return platform.dist()[0]
 
 from .repos import Repo
-from .libkas import run_cmd
+from .libkas import run_cmd, repo_fetch, repo_checkout
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -329,8 +329,10 @@ class ConfigStatic(Config):
             missing_repos_old = missing_repos
             if missing_repos:
                 complete = False
-                self._fetch_missing_repos(missing_repos)
                 repo_dict = self.get_repo_dict()
+                for repo in missing_repos:
+                    repo_fetch(self, repo_dict[repo])
+                    repo_checkout(self, repo_dict[repo])
                 repos = {r: repo_dict[r].path for r in repo_dict}
 
     def get_repos(self):
@@ -387,40 +389,6 @@ class ConfigStatic(Config):
             repo_dict[repo] = rep
         return repo_dict
 
-    def _fetch_missing_repos(self, missing_repos):
-        """
-            Fetches all repos from the missing_repos list.
-        """
-        from .libcmds import (Macro, ReposFetch, ReposCheckout)
-
-        class MissingRepoConfig(Config):
-            """
-                Custom config class, because we only want to
-                fetch the missing repositories needed for a
-                complete configuration not all of them.
-            """
-            def __init__(self, outerself):
-                super().__init__()
-                self.outerself = outerself
-                self.filename = outerself.filename
-                self.environ = outerself.environ
-                self.__kas_work_dir = outerself.kas_work_dir
-
-            def get_repo_ref_dir(self):
-                return self.outerself.get_repo_ref_dir()
-
-            def get_proxy_config(self):
-                return self.outerself.get_proxy_config()
-
-            def get_repos(self):
-                return list(map(lambda x: self.outerself.get_repo_dict()[x],
-                                missing_repos))
-
-        macro = Macro()
-        macro.add(ReposFetch())
-        macro.add(ReposCheckout())
-        macro.run(MissingRepoConfig(self))
-
 
 def load_config(filename, target):
     """

kas/libcmds.py

@@ -28,7 +28,7 @@ import logging
 import shutil
 import os
 from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check,
-                     run_cmd, get_build_environ)
+                     get_build_environ, repo_fetch, repo_checkout)
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -210,42 +210,7 @@ class ReposFetch(Command):
 
     def execute(self, config):
         for repo in config.get_repos():
-            if repo.git_operation_disabled:
+            repo_fetch(config, repo)
-                continue
-
-            if not os.path.exists(repo.path):
-                os.makedirs(os.path.dirname(repo.path), exist_ok=True)
-                gitsrcdir = os.path.join(config.get_repo_ref_dir() or '',
-                                         repo.qualified_name)
-                logging.debug('Looking for repo ref dir in %s', gitsrcdir)
-                if config.get_repo_ref_dir() and os.path.exists(gitsrcdir):
-                    run_cmd(['/usr/bin/git',
-                             'clone',
-                             '--reference', gitsrcdir,
-                             repo.url, repo.path],
-                            env=config.environ,
-                            cwd=config.kas_work_dir)
-                else:
-                    run_cmd(['/usr/bin/git', 'clone', '-q', repo.url,
-                             repo.path],
-                            env=config.environ,
-                            cwd=config.kas_work_dir)
-                continue
-
-            # Does refspec in the current repository?
-            (retc, output) = run_cmd(['/usr/bin/git', 'cat-file',
-                                      '-t', repo.refspec], env=config.environ,
-                                     cwd=repo.path, fail=False)
-            if retc == 0:
-                continue
-
-            # No it is missing, try to fetch
-            (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
-                                     env=config.environ,
-                                     cwd=repo.path, fail=False)
-            if retc:
-                logging.warning('Could not update repository %s: %s',
-                                repo.name, output)
 
 
 class ReposCheckout(Command):
@@ -258,27 +223,4 @@ class ReposCheckout(Command):
 
     def execute(self, config):
         for repo in config.get_repos():
-            if repo.git_operation_disabled:
+            repo_checkout(config, repo)
-                continue
-
-            # Check if repos is dirty
-            (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
-                                  env=config.environ, cwd=repo.path,
-                                  fail=False)
-            if len(output):
-                logging.warning('Repo %s is dirty. no checkout', repo.name)
-                continue
-
-            # Check if current HEAD is what in the config file is defined.
-            (_, output) = run_cmd(['/usr/bin/git', 'rev-parse',
-                                   '--verify', 'HEAD'],
-                                  env=config.environ, cwd=repo.path)
-
-            if output.strip() == repo.refspec:
-                logging.info('Repo %s has already checkout out correct '
-                             'refspec. nothing to do', repo.name)
-                continue
-
-            run_cmd(['/usr/bin/git', 'checkout', '-q',
-                     '{refspec}'.format(refspec=repo.refspec)],
-                    cwd=repo.path)

kas/libkas.py

@@ -158,6 +158,78 @@ def find_program(paths, name):
     return None
 
 
+def repo_fetch(config, repo):
+    """
+        Fetches the repository to the kas_work_dir.
+    """
+    if repo.git_operation_disabled:
+        return
+
+    if not os.path.exists(repo.path):
+        os.makedirs(os.path.dirname(repo.path), exist_ok=True)
+        gitsrcdir = os.path.join(config.get_repo_ref_dir() or '',
+                                 repo.qualified_name)
+        logging.debug('Looking for repo ref dir in %s', gitsrcdir)
+        if config.get_repo_ref_dir() and os.path.exists(gitsrcdir):
+            run_cmd(['/usr/bin/git',
+                     'clone',
+                     '--reference', gitsrcdir,
+                     repo.url, repo.path],
+                    env=config.environ,
+                    cwd=config.kas_work_dir)
+        else:
+            run_cmd(['/usr/bin/git', 'clone', '-q', repo.url,
+                     repo.path],
+                    env=config.environ,
+                    cwd=config.kas_work_dir)
+        return
+
+    # Does refspec in the current repository?
+    (retc, output) = run_cmd(['/usr/bin/git', 'cat-file',
+                              '-t', repo.refspec], env=config.environ,
+                             cwd=repo.path, fail=False)
+    if retc == 0:
+        return
+
+    # No it is missing, try to fetch
+    (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
+                             env=config.environ,
+                             cwd=repo.path, fail=False)
+    if retc:
+        logging.warning('Could not update repository %s: %s',
+                        repo.name, output)
+
+
+def repo_checkout(config, repo):
+    """
+        Checks out the correct revision of the repo.
+    """
+    if repo.git_operation_disabled:
+        return
+
+    # Check if repos is dirty
+    (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
+                          env=config.environ, cwd=repo.path,
+                          fail=False)
+    if len(output):
+        logging.warning('Repo %s is dirty. no checkout', repo.name)
+        return
+
+    # Check if current HEAD is what in the config file is defined.
+    (_, output) = run_cmd(['/usr/bin/git', 'rev-parse',
+                           '--verify', 'HEAD'],
+                          env=config.environ, cwd=repo.path)
+
+    if output.strip() == repo.refspec:
+        logging.info('Repo %s has already checkout out correct '
+                     'refspec. nothing to do', repo.name)
+        return
+
+    run_cmd(['/usr/bin/git', 'checkout', '-q',
+             '{refspec}'.format(refspec=repo.refspec)],
+            cwd=repo.path)
+
+
 def get_build_environ(config, build_dir):
     """
         Create the build environment variables.