implements patch support for repos

With this commit its now possible to patch 3rd party repos before bitbake is started.

Example:

This is our repo:
  .
  ├── kas.yml
  ├── first-patch.patch
  └── quilt-patches
      ├── second-patch.patch
      ├── third-patch.patch
      └── series

Content of kas.yml:
  header:
    version: 8

  repos:
    my:
    third-party:
      url: "git://example.com/third-party.git"
      refspec: "35adf4...34"
      patches:
        01-first:
          repo: my
          path: "first-patch.patch"
        02-second:
          repo: my
          path: "quilt-patches"

Currently only 'git' repositories can be patched.

Signed-off-by: Claudius Heine <ch@denx.de>

docs/format-changelog.rst

@@ -66,3 +66,12 @@ Added
 
 - ``type`` property to ``repos`` to be able to express which version control
   system to use.
+
+Version 8
+---------
+
+Added
+~~~~~
+
+- ``patches`` property to ``repos`` to be able to apply additional patches to
+  the repo.

docs/userguide.rst

@@ -352,6 +352,22 @@ Configuration reference
           '0', 'false']``. This way it is possible to overwrite the inclusion
           of a layer in latter loaded configuration files.
 
+    * ``patches``: dict [optional]
+        Contains the patches that should be applied to this repo before it is
+        used.
+
+      * ``<patches-id>``: dict [optional]
+          One entry in patches with its specific and unique id. All available
+          patch entries are applied in the order of their sorted
+          ``<patches-id>``.
+
+        * ``repo``: string [required]
+            The identifier of the repo where the path of this entry is relative
+            to.
+
+        * ``path``: string [required]
+            The path to one patch file or a quilt formatted patchset directory.
+
 * ``bblayers_conf_header``: dict [optional]
     This contains strings that should be added to the ``bblayers.conf`` before
     any layers are included.

kas/__version__.py

@@ -28,5 +28,5 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
 __version__ = '0.17.0'
 
 # Please update docs/format-changelog.rst when changing the file version.
-__file_version__ = 7
+__file_version__ = 8
 __compatible_file_version__ = 1

kas/build.py

@@ -29,7 +29,7 @@ from .libkas import find_program, run_cmd, kasplugin
 from .libcmds import (Macro, Command, SetupDir, SetupProxy,
                       CleanupSSHAgent, SetupSSHAgent, SetupEnviron,
                       WriteConfig, SetupHome, ReposFetch,
-                      ReposCheckout)
+                      ReposApplyPatches, ReposCheckout)
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -86,11 +86,12 @@ class Build:
         macro.add(ReposFetch())
         macro.add(ReposCheckout())
         macro.add(SetupEnviron())
+        macro.add(SetupHome())
+        macro.add(ReposApplyPatches())
 
         macro.add(WriteConfig())
 
         # Build
-        macro.add(SetupHome())
         macro.add(BuildCommand(args.task))
 
         if 'SSH_PRIVATE_KEY' in os.environ:

kas/configschema.py

@@ -142,6 +142,29 @@ CONFIGSCHEMA = {
                                     ],
                                 },
                             },
+                            'patches': {
+                                'type': 'object',
+                                'additionalProperties': {
+                                    'oneOf': [
+                                        {
+                                            'type': 'object',
+                                            'additionalProperties': False,
+                                            'required': ['repo', 'path'],
+                                            'properties': {
+                                                'repo': {
+                                                    'type': 'string'
+                                                },
+                                                'path': {
+                                                    'type': 'string'
+                                                },
+                                            },
+                                        },
+                                        {
+                                            'type': 'null'
+                                        },
+                                    ],
+                                },
+                            },
                         },
                     },
                     {

kas/libcmds.py

@@ -28,7 +28,7 @@ import logging
 import shutil
 import os
 from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check,
-                     get_build_environ, repos_fetch)
+                     get_build_environ, repos_fetch, repos_apply_patches)
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -92,6 +92,8 @@ class SetupHome(Command):
             fds.write('\n')
         with open(self.tmpdirname + '/.netrc', 'w') as fds:
             fds.write('\n')
+        shutil.copyfile(os.path.expanduser('~/.gitconfig'),
+                        self.tmpdirname + '/.gitconfig')
         config.environ['HOME'] = self.tmpdirname
 
 
@@ -204,6 +206,18 @@ class ReposFetch(Command):
         repos_fetch(config, config.get_repos())
 
 
+class ReposApplyPatches(Command):
+    """
+        Applies the patches defined in the configuration to the repositories.
+    """
+
+    def __str__(self):
+        return 'repos_apply_patches'
+
+    def execute(self, config):
+        repos_apply_patches(config, config.get_repos())
+
+
 class ReposCheckout(Command):
     """
         Ensures that the right revision of each repo is check out.

kas/libkas.py

@@ -184,6 +184,27 @@ def repos_fetch(config, repos):
             sys.exit(task.result())
 
 
+def repos_apply_patches(config, repos):
+    """
+        Applies the patches to the repositories.
+    """
+    tasks = []
+    for repo in repos:
+        if not hasattr(asyncio, 'ensure_future'):
+            # pylint: disable=no-member,deprecated-method
+            task = asyncio.async(repo.apply_patches_async(config))
+        else:
+            task = asyncio.ensure_future(repo.apply_patches_async(config))
+        tasks.append(task)
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(asyncio.wait(tasks))
+
+    for task in tasks:
+        if task.result():
+            sys.exit(task.result())
+
+
 def get_build_environ(config, build_dir):
     """
         Create the build environment variables.

kas/repos.py

@@ -38,12 +38,14 @@ class Repo:
         Represents a repository in the kas configuration.
     """
 
-    def __init__(self, url, path, refspec, layers, disable_operations):
+    def __init__(self, url, path, refspec, layers, patches,
+                 disable_operations):
         # pylint: disable=too-many-arguments
         self.url = url
         self.path = path
         self.refspec = refspec
         self._layers = layers
+        self._patches = patches
         self.name = os.path.basename(self.path)
         self.operations_disabled = disable_operations
 
@@ -78,6 +80,15 @@ class Repo:
                              str(laydict[x]).lower() not in
                              ['disabled', 'excluded', 'n', 'no', '0', 'false'],
                              layers_dict))
+        patches_dict = repo_config.get('patches', {})
+        patches = list(
+            {
+                'id': p,
+                'repo': patches_dict[p]['repo'],
+                'path': patches_dict[p]['path'],
+            }
+            for p in sorted(patches_dict)
+            if patches_dict[p])
         url = repo_config.get('url', None)
         name = repo_config.get('name', name)
         typ = repo_config.get('type', 'git')
@@ -104,9 +115,10 @@ class Repo:
                     path = os.path.join(config.kas_work_dir, path)
 
         if typ == 'git':
-            return GitRepo(url, path, refspec, layers, disable_operations)
+            return GitRepo(url, path, refspec, layers, patches,
+                           disable_operations)
         if typ == 'hg':
-            return MercurialRepo(url, path, refspec, layers,
+            return MercurialRepo(url, path, refspec, layers, patches,
                                  disable_operations)
         raise NotImplementedError('Repo typ "%s" not supported.' % typ)
 
@@ -209,6 +221,58 @@ class RepoImpl(Repo):
 
         run_cmd(self.checkout_cmd(), cwd=self.path)
 
+    @asyncio.coroutine
+    def apply_patches_async(self, config):
+        """
+            Applies patches to repository asynchronously.
+        """
+        if self.operations_disabled:
+            return 0
+
+        for patch in self._patches:
+            other_repo = config.repo_dict.get(patch['repo'], None)
+
+            if not other_repo:
+                logging.warning('Could not find referenced repo. '
+                                '(missing repo: %s, repo: %s, '
+                                'patch entry: %s)',
+                                patch['repo'],
+                                self.name,
+                                patch['id'])
+                continue
+
+            path = os.path.join(other_repo.path, patch['path'])
+            cmd = []
+
+            if os.path.isfile(path):
+                cmd = self.apply_patches_file_cmd(path)
+            elif os.path.isdir(path):
+                cmd = self.apply_patches_quilt_cmd(path)
+            else:
+                logging.warning('Could not find patch. '
+                                '(patch path: %s, repo: %s, patch entry: %s)',
+                                path,
+                                self.name,
+                                patch['id'])
+                continue
+
+            (retc, output) = yield from run_cmd_async(cmd,
+                                                      env=config.environ,
+                                                      cwd=self.path,
+                                                      fail=False)
+
+            if retc:
+                logging.error('Could not apply patch. Please fix repos and '
+                              'patches. (patch path: %s, repo: %s, patch '
+                              'entry: %s, vcs output: %s)',
+                              path, self.name, patch['id'], output)
+                return 1
+            else:
+                logging.info('Patch applied. '
+                             '(patch path: %s, repo: %s, patch entry: %s)',
+                             path, self.name, patch['id'])
+        return 0
+
 
 class GitRepo(RepoImpl):
     """
@@ -238,6 +302,13 @@ class GitRepo(RepoImpl):
         return ['git', 'checkout', '-q',
                 '{refspec}'.format(refspec=self.refspec)]
 
+    def apply_patches_file_cmd(self, path):
+        return ['git', 'am', '-q', path]
+
+    def apply_patches_quilt_cmd(self, path):
+        return ['git', 'quiltimport', '--author', 'kas <kas@example.com>',
+                '--patches', path]
+
 
 class MercurialRepo(RepoImpl):
     """
@@ -262,3 +333,9 @@ class MercurialRepo(RepoImpl):
 
     def checkout_cmd(self):
         return ['hg', 'checkout', '{refspec}'.format(refspec=self.refspec)]
+
+    def apply_patches_file_cmd(self, path):
+        raise NotImplementedError()
+
+    def apply_patches_quilt_cmd(self, path):
+        raise NotImplementedError()

kas/shell.py

@@ -30,7 +30,7 @@ from kas.libkas import kasplugin
 from kas.config import Config
 from kas.libcmds import (Macro, Command, SetupDir, SetupProxy, SetupEnviron,
                          WriteConfig, SetupHome, ReposFetch, ReposCheckout,
-                         CleanupSSHAgent, SetupSSHAgent)
+                         ReposApplyPatches, CleanupSSHAgent, SetupSSHAgent)
 
 __license__ = 'MIT'
 __copyright__ = 'Copyright (c) Siemens AG, 2017'
@@ -91,13 +91,15 @@ class Shell:
         if not args.keep_config_unchanged:
             macro.add(ReposFetch())
             macro.add(ReposCheckout())
-            macro.add(SetupEnviron())
+
+        macro.add(SetupEnviron())
+        macro.add(SetupHome())
+
+        if not args.keep_config_unchanged:
+            macro.add(ReposApplyPatches())
             macro.add(WriteConfig())
-        else:
-            macro.add(SetupEnviron())
 
         # Shell
-        macro.add(SetupHome())
         macro.add(ShellCommand(args.command))
 
         if 'SSH_PRIVATE_KEY' in os.environ: