Initial public release

This is the first public release of kas.

Signed-off-by: Daniel Wagner <daniel.wagner@siemens.com>

kas/__init__.py

@@ -0,0 +1,24 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'

kas/__main__.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from .kas import main
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+main()

kas/__version__.py

@@ -0,0 +1 @@
+__version__ = '0.9.0'

kas/build.py

@@ -0,0 +1,97 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+from .config import load_config
+from .libkas import find_program, run_cmd
+from .libcmds import (Macro, Command, SetupDir, SetupProxy,
+                      CleanupSSHAgent, SetupSSHAgent, SetupEnviron,
+                      WriteConfig, SetupHome, ReposFetch,
+                      ReposCheckout)
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class Build:
+    def __init__(self, parser):
+        bld_psr = parser.add_parser('build')
+
+        bld_psr.add_argument('config',
+                             help='Config file')
+        bld_psr.add_argument('--target',
+                             help='Select target to build',
+                             default='core-image-minimal')
+        bld_psr.add_argument('--task',
+                             help='Select which task should be executed',
+                             default='build')
+        bld_psr.add_argument('--skip',
+                             help='Skip build steps',
+                             default=[])
+
+    def run(self, args):
+        if args.cmd != 'build':
+            return False
+
+        cfg = load_config(args.config, args.target)
+
+        macro = Macro()
+
+        # Prepare
+        macro.add(SetupDir())
+        macro.add(SetupProxy())
+
+        if 'SSH_PRIVATE_KEY' in os.environ:
+            macro.add(SetupSSHAgent())
+
+        macro.add(ReposFetch())
+        macro.add(ReposCheckout())
+        macro.add(SetupEnviron())
+
+        # TODO if cfg.has_changed() was not working properly
+        macro.add(WriteConfig())
+
+        # Build
+        macro.add(SetupHome())
+        macro.add(BuildCommand(args.task))
+
+        if 'SSH_PRIVATE_KEY' in os.environ:
+            macro.add(CleanupSSHAgent())
+
+        macro.run(cfg, args.skip)
+
+        return True
+
+
+class BuildCommand(Command):
+    def __init__(self, task):
+        Command.__init__
+        self.task = task
+
+    def __str__(self):
+        return 'build'
+
+    def execute(self, config):
+        # Start bitbake build of image
+        bitbake = find_program(config.environ['PATH'], 'bitbake')
+        run_cmd([bitbake, '-k', config.get_bitbake_target(), '-c', self.task],
+                env=config.environ, cwd=config.build_dir)

kas/config.py

@@ -0,0 +1,344 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import sys
+import logging
+import errno
+import json
+import platform
+import hashlib
+import yaml
+from .repos import Repo
+from .kasstate import KasState
+from .libkas import run_cmd
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class Config:
+    def __init__(self):
+        self.state = KasState()
+
+    def _get_prop(self, item, default):
+        return self.state.get_option(self.section, item, default)
+
+    def _set_prop(self, item, value):
+        self.state.set_option(self.section, item, value)
+
+    @property
+    def section(self):
+        return os.path.realpath(self.filename)
+
+    @property
+    def build_dir(self):
+        return self._get_prop('build_dir', os.getcwd() + '/build')
+
+    @build_dir.setter
+    def build_dir(self, value):
+        self._set_prop('build_dir', value)
+
+    @property
+    def kas_work_dir(self):
+        return self._get_prop('kas_work_dir', os.getcwd())
+
+    @kas_work_dir.setter
+    def kas_work_dir(self, value):
+        self._set_prop('kas_work_dir', value)
+
+    @property
+    def hash(self):
+        return self._get_prop('config_hash', 0)
+
+    @hash.setter
+    def hash(self, value):
+        self._set_prop('config_hash', value)
+
+    def setup_environ(self):
+        (distro, version, id) = platform.dist()
+        if distro in ['fedora', 'SuSE']:
+            self.environ = {'LC_ALL': 'en_US.utf8',
+                            'LANG': 'en_US.utf8',
+                            'LANGUAGE': 'en_US'}
+        elif distro in ['Ubuntu', 'debian']:
+            self.environ = {'LC_ALL': 'en_US.UTF-8',
+                            'LANG': 'en_US.UTF-8',
+                            'LANGUAGE': 'en_US:en'}
+        else:
+            logging.warning('kas: Unsupported distro. No default locales set.')
+            self.environ = {}
+
+    def get_repo_ref_dir(self):
+        return os.environ.get('KAS_REPO_REF_DIR', None)
+
+    def has_changed(self):
+        return self.config_hash != self.hash
+
+
+class ConfigPython(Config):
+    def __init__(self, filename, target):
+        super().__init__()
+        self.filename = os.path.abspath(filename)
+        try:
+            with open(self.filename) as file:
+                env = {}
+                data = file.read()
+                self.hash = hashlib.sha512(str(data).encode('utf-8')).hexdigest()
+                exec(data, env)
+                self._config = env
+        except IOError:
+            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT),
+                          self.filename)
+
+        self.create_config(target)
+        self.setup_environ()
+
+    def __str__(self):
+        s = 'target: {}\n'.format(self.target)
+        s += 'repos:\n'
+        for r in self.get_repos():
+            s += '  {}\n'.format(r.__str__())
+        s += 'environ:\n'
+        for k, v in self.environ.items():
+            s += '  {} = {}\n'.format(k, v)
+        s += 'proxy:\n'
+        for k, v in self.get_proxy_config().items():
+            s += '  {} = {}\n'.format(k, v)
+        return s
+
+    def pre_hook(self, fname):
+        try:
+            self._config[fname + '_prepend'](self)
+        except KeyError:
+            pass
+
+    def post_hook(self, fname):
+        try:
+            self._config[fname + '_append'](self)
+        except KeyError:
+            pass
+
+    def get_hook(self, fname):
+        try:
+            return self._config[fname]
+        except KeyError:
+            return None
+
+    def create_config(self, target):
+        self.target = target
+        self.repos = self._config['get_repos'](self, target)
+
+    def get_proxy_config(self):
+        return self._config['get_proxy_config']()
+
+    def get_repos(self):
+        return iter(self.repos)
+
+    def get_target(self):
+        return self.target
+
+    def get_bitbake_target(self):
+        try:
+            return self._config['get_bitbake_target'](self)
+        except KeyError:
+            return self.target
+
+    def get_bblayers_conf_header(self):
+        try:
+            return self._config['get_bblayers_conf_header']()
+        except KeyError:
+            return ''
+
+    def get_local_conf_header(self):
+        try:
+            return self._config['get_local_conf_header']()
+        except:
+            return ''
+
+    def get_machine(self):
+        try:
+            return self._config['get_machine'](self)
+        except KeyError:
+            return 'qemu'
+
+    def get_distro(self):
+        try:
+            return self._config['get_distro'](self)
+        except KeyError:
+            return 'poky'
+
+    def get_gitlabci_config(self):
+        try:
+            return self._config['get_gitlabci_config'](self)
+        except KeyError:
+            return ''
+
+
+class ConfigStatic(Config):
+    def __init__(self, filename, target):
+        super().__init__()
+        self.filename = os.path.abspath(filename)
+        self._config = []
+
+    def pre_hook(self, target):
+        pass
+
+    def post_hook(self, target):
+        pass
+
+    def get_hook(self, fname):
+        return None
+
+    def get_proxy_config(self):
+        try:
+            return self._config['proxy_config']
+        except KeyError:
+            return {'http_proxy': os.environ.get('http_proxy', ''),
+                    'https_proxy': os.environ.get('https_proxy', ''),
+                    'no_proxy': os.environ.get('no_proxy', '')}
+
+    def get_repos(self):
+        repos = []
+        for repo in self._config['repos']:
+            try:
+                sublayers = repo['sublayers']
+            except KeyError:
+                sublayers = None
+
+            url = repo['url']
+            if url == '':
+                # in-tree configuration
+                (rc, output) = run_cmd(['/usr/bin/git',
+                                        'rev-parse',
+                                        '--show-toplevel'],
+                                        cwd=os.path.dirname(self.filename),
+                                        env=self.environ)
+                url = output.strip()
+                r = Repo(url=url,
+                         path=url,
+                         sublayers=sublayers)
+                r.disable_git_operations()
+            else:
+                name = os.path.basename(os.path.splitext(url)[0])
+                r = Repo(url=url,
+                         path=os.path.join(self.kas_work_dir, name),
+                         refspec=repo['refspec'],
+                         sublayers=sublayers)
+            repos.append(r)
+
+        return repos
+
+    def get_bitbake_target(self):
+        try:
+            return self._config['target']
+        except KeyError:
+            return 'core-image-minimal'
+
+    def get_bblayers_conf_header(self):
+        try:
+            return self._config['bblayers_conf_header']
+        except KeyError:
+            return ''
+
+    def get_local_conf_header(self):
+        try:
+            return self._config['local_conf_header']
+        except KeyError:
+            return ''
+
+    def get_machine(self):
+        try:
+            return self._config['machine']
+        except KeyError:
+            return 'qemu'
+
+    def get_distro(self):
+        try:
+            return self._config['distro']
+        except KeyError:
+            return 'poky'
+
+    def get_gitlabci_config(self):
+        try:
+            return self._config['gitlabci_config']
+        except KeyError:
+            return ''
+
+
+class ConfigJson(ConfigStatic):
+    def __init__(self, filename, target):
+        super().__init__(filename, target)
+        self.filename = os.path.abspath(filename)
+        try:
+            with open(self.filename, 'r') as f:
+                self.hash = hashlib.sha512(f.read().encode('utf-8')).hexdigest()
+                f.seek(0)
+                self._config = json.load(f)
+        except json.decoder.JSONDecodeError  as msg:
+            logging.error('Could not load JSON config: {}'.format(msg))
+            sys.exit(1)
+        self.setup_environ()
+
+    def get_bblayers_conf_header(self):
+        list = super().get_bblayers_conf_header()
+        conf = ''
+        for line in list:
+            conf += str(line) + '\n'
+        return conf
+
+    def get_local_conf_header(self):
+        list = super().get_local_conf_header()
+        conf = ''
+        for line in list:
+            conf += str(line) + '\n'
+        return conf
+
+
+class ConfigYaml(ConfigStatic):
+    def __init__(self, filename, target):
+        super().__init__(filename, target)
+        self.filename = os.path.abspath(filename)
+        try:
+            with open(self.filename, 'r') as f:
+                self.hash = hashlib.sha512(f.read().encode('utf-8')).hexdigest()
+                f.seek(0)
+                self._config = yaml.load(f)
+        except yaml.loader.ParserError as msg:
+            logging.error('Could not load YAML config: {}'.format(msg))
+            sys.exit(1)
+        self.setup_environ()
+
+
+def load_config(filename, target):
+    f, ext = os.path.splitext(filename)
+    if ext == '.py':
+        cfg = ConfigPython(filename, target)
+    elif ext == '.json':
+        cfg = ConfigJson(filename, target)
+    elif ext == '.yml':
+        cfg = ConfigYaml(filename, target)
+    else:
+        logging.error('Config file extenstion not recognized')
+        sys.exit(1)
+
+    return cfg

kas/kas.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+#
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import argparse
+import traceback
+import logging
+import sys
+import os
+import pkg_resources
+
+try:
+    import colorlog
+    have_colorlog = True
+except ImportError:
+    have_colorlog = False
+
+from .build import Build
+from .shell import Shell
+from .__version__ import __version__
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+def create_logger():
+    log = logging.getLogger()  # root logger
+    log.setLevel(logging.INFO)
+    format = '%(asctime)s - %(levelname)-8s - %(message)s'
+    date_format = '%Y-%m-%d %H:%M:%S'
+    if have_colorlog and os.isatty(2):
+        cformat = '%(log_color)s' + format
+        colors = {'DEBUG': 'reset',
+                  'INFO': 'reset',
+                  'WARNING': 'bold_yellow',
+                  'ERROR': 'bold_red',
+                  'CRITICAL': 'bold_red'}
+        f = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
+    else:
+        f = logging.Formatter(format, date_format)
+    ch = logging.StreamHandler()
+    ch.setFormatter(f)
+    log.addHandler(ch)
+    return logging.getLogger(__name__)
+
+
+def kas(argv):
+    create_logger()
+
+    parser = argparse.ArgumentParser(description='Steer ebs-yocto builds')
+
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s ' + __version__)
+
+    parser.add_argument('-d', '--debug',
+                        action='store_true',
+                        help='Enable debug logging')
+
+    subparser = parser.add_subparsers(help='sub command help', dest='cmd')
+    sub_cmds = [Build(subparser), Shell(subparser)]
+
+    for plugin in pkg_resources.iter_entry_points('kas.plugins'):
+        cmd = plugin.load()
+        sub_cmds.append(cmd(subparser))
+
+    args = parser.parse_args(argv)
+
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    for cmd in sub_cmds:
+        if cmd.run(args):
+            break
+
+
+def main():
+    try:
+        sys.exit(kas(sys.argv[1:]))
+    except Exception as err:
+        logging.error('%s' % err)
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

kas/kasstate.py

@@ -0,0 +1,57 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import codecs
+import os
+from configparser import SafeConfigParser
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class KasState:
+    def __init__(self):
+        self.filename = os.path.join(os.environ.get('XDG_CONFIG_HOME',
+                                                    os.path.expandvars('$HOME/.config')),
+                                     os.path.join('kas', 'kasstate.ini'))
+        self.parser = SafeConfigParser()
+        try:
+            with codecs.open(self.filename, 'r', encoding='utf-8') as f:
+                self.parser.readfp(f)
+        except:
+            pass
+
+    def __del__(self):
+        if not os.path.exists(os.path.dirname(self.filename)):
+            os.makedirs(os.path.dirname(self.filename))
+        with codecs.open(self.filename, 'w', encoding='utf-8') as f:
+            self.parser.write(f)
+
+    def get_option(self, section, option, default):
+        if not self.parser.has_option(section, option):
+            self.set_option(section, option, default)
+        return self.parser.get(section, option)
+
+    def set_option(self, section, option, value):
+        if not self.parser.has_section(section):
+            self.parser.add_section(section)
+        self.parser.set(section, option, value)

kas/libcmds.py

@@ -0,0 +1,233 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import tempfile
+import logging
+import shutil
+import os
+from urllib.parse import urlparse
+from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check,
+                     run_cmd, get_oe_environ)
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class Macro:
+    def __init__(self):
+        self.commands = []
+
+    def add(self, command):
+        self.commands.append(command)
+
+    def run(self, config, skip=[]):
+        for c in self.commands:
+            name = str(c)
+            if name in skip:
+                continue
+            pre = config.pre_hook(name)
+            if pre:
+                logging.debug('execute ' + pre)
+                pre(config)
+            cmd = config.get_hook(name)
+            if cmd:
+                logging.debug('execute ' + cmd)
+                cmd(config)
+            else:
+                logging.debug('execute ' + str(c))
+                c.execute(config)
+            post = config.post_hook(name)
+            if post:
+                logging.debug('execute ' + post)
+                post(config)
+
+
+class Command:
+    def execute(self, config):
+        pass
+
+
+class SetupHome(Command):
+    def __init__(self):
+        self.tmpdirname = tempfile.mkdtemp()
+
+    def __del__(self):
+        shutil.rmtree(self.tmpdirname)
+
+    def __str__(self):
+        return 'setup_home'
+
+    def execute(self, config):
+        with open(self.tmpdirname + '/.wgetrc', 'w') as f:
+            f.write('\n')
+        with open(self.tmpdirname + '/.netrc', 'w') as f:
+            f.write('\n')
+        config.environ['HOME'] = self.tmpdirname
+
+
+class SetupDir(Command):
+    def __str__(self):
+        return 'setup_dir'
+
+    def execute(self, config):
+        os.chdir(config.kas_work_dir)
+        if not os.path.exists(config.build_dir):
+            os.mkdir(config.build_dir)
+
+
+class SetupSSHAgent(Command):
+    def __str__(self):
+        return 'setup_ssh_agent'
+
+    def execute(self, config):
+        ssh_setup_agent(config)
+        ssh_no_host_key_check(config)
+
+
+class CleanupSSHAgent(Command):
+    """Remove all the identities and stop the ssh-agent instance"""
+
+    def __str__(self):
+        return 'cleanup_ssh_agent'
+
+    def execute(self, config):
+        ssh_cleanup_agent(config)
+
+
+class SetupProxy(Command):
+    def __str__(self):
+        return 'setup_proxy'
+
+    def execute(self, config):
+        config.environ.update(config.get_proxy_config())
+
+
+class SetupEnviron(Command):
+    def __str__(self):
+        return 'setup_environ'
+
+    def execute(self, config):
+        config.environ.update(get_oe_environ(config, config.build_dir))
+
+
+class WriteConfig(Command):
+    def __str__(self):
+        return 'write_config'
+
+    def execute(self, config):
+        self._write_bblayers_conf(config)
+        self._write_local_conf(config)
+
+    def _append_layers(self, config, file):
+        for repo in config.get_repos():
+            file.write(' \\\n'.join(repo.layers + ['']))
+
+    def _write_bblayers_conf(self, config):
+        filename = config.build_dir + '/conf/bblayers.conf'
+        with open(filename, 'w') as file:
+            file.write(config.get_bblayers_conf_header())
+            file.write('BBLAYERS ?= " \\\n')
+            self._append_layers(config, file)
+            file.write('"\n')
+
+    def _write_local_conf(self, config):
+        filename = config.build_dir + '/conf/local.conf'
+        with open(filename, 'w') as file:
+            file.write(config.get_local_conf_header())
+            file.write('MACHINE ?= "{}"\n'.format(config.get_machine()))
+            file.write('DISTRO ?= "{}"\n'.format(config.get_distro()))
+
+
+class ReposFetch(Command):
+    def __str__(self):
+        return 'repos_fetch'
+
+    def execute(self, config):
+        for repo in config.get_repos():
+            if repo.git_operation_disabled:
+                continue
+
+            if not os.path.exists(repo.path):
+                os.makedirs(os.path.dirname(repo.path), exist_ok=True)
+                gitsrcdir = os.path.join(config.get_repo_ref_dir() or '',
+                                         repo.qualified_name)
+                logging.debug('Looking for repo ref dir in {}'.format(gitsrcdir))
+                if config.get_repo_ref_dir() and os.path.exists(gitsrcdir):
+                    run_cmd(['/usr/bin/git',
+                             'clone',
+                             '--reference', gitsrcdir,
+                             repo.url, repo.path],
+                            env=config.environ,
+                            cwd=config.kas_work_dir)
+                else:
+                    run_cmd(['/usr/bin/git', 'clone', '-q', repo.url, repo.path],
+                            env=config.environ,
+                            cwd=config.kas_work_dir)
+                continue
+
+            # Does refspec in the current repository?
+            (rc, output) = run_cmd(['/usr/bin/git', 'cat-file',
+                                    '-t', repo.refspec], env=config.environ,
+                                   cwd=repo.path, fail=False)
+            if rc == 0:
+                continue
+
+            # No it is missing, try to fetch
+            (rc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
+                                   env=config.environ,
+                                   cwd=repo.path, fail=False)
+            if rc:
+                logging.warning('Could not update repository {}: {}'.
+                                format(repo.name, output))
+
+
+class ReposCheckout(Command):
+    def __str__(self):
+        return 'repos_checkout'
+
+    def execute(self, config):
+        for repo in config.get_repos():
+            if repo.git_operation_disabled:
+                continue
+
+            # Check if repos is dirty
+            (rc, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
+                                   env=config.environ, cwd=repo.path,
+                                   fail=False)
+            if len(output):
+                logging.warning('Repo {} is dirty. no checkout'.
+                                format(repo.name))
+                continue
+
+            # Check if current HEAD is what in the config file is defined.
+            (rc, output) = run_cmd(['/usr/bin/git', 'rev-parse',
+                                    '--verify', 'HEAD'], env=config.environ,
+                                   cwd=repo.path)
+
+            if output.strip() == repo.refspec:
+                logging.info(('Repo {} has already checkout out correct '
+                              'refspec. nothing to do').format(repo.name))
+                continue
+
+            run_cmd(['/usr/bin/git', 'checkout', '-q',
+                     '{refspec}'.format(refspec=repo.refspec)],
+                    cwd=repo.path)

kas/libkas.py

@@ -0,0 +1,223 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import re
+import os
+import sys
+import logging
+import tempfile
+import asyncio
+from subprocess import Popen, PIPE
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class LogOutput:
+    def __init__(self, live):
+        self.live = live
+        self.stdout = []
+        self.stderr = []
+
+    def log_stdout(self, line):
+        if self.live:
+            logging.info(line.strip())
+        self.stdout.append(line)
+
+    def log_stderr(self, line):
+        if self.live:
+            logging.error(line.strip())
+        self.stderr.append(line)
+
+
+@asyncio.coroutine
+def _read_stream(stream, cb):
+    while True:
+        line = yield from stream.readline()
+        try:
+            line = line.decode('utf-8')
+        except:
+            logging.warning('Could not decode line from stream - ignore it')
+        if line:
+            cb(line)
+        else:
+            break
+
+@asyncio.coroutine
+def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb):
+    if shell:
+        process = yield from asyncio.create_subprocess_shell(
+            cmd,
+            env=env,
+            cwd=cwd,
+            universal_newlines=True,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE)
+    else:
+        process = yield from asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=cwd,
+            env=env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE)
+
+    yield from asyncio.wait([
+        _read_stream(process.stdout, stdout_cb),
+        _read_stream(process.stderr, stderr_cb)
+    ])
+    ret = yield from process.wait()
+    return ret
+
+
+def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True):
+    rc = 0
+    stdout = []
+    stderr = []
+    cmdstr = cmd
+    if not shell:
+        cmdstr = ' '.join(cmd)
+    logging.info('{}$ {}'.format(cwd, cmdstr))
+
+    logo = LogOutput(liveupdate)
+    if asyncio.get_event_loop().is_closed():
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+    else:
+        loop = asyncio.get_event_loop()
+
+    rc = loop.run_until_complete(
+        _stream_subprocess(cmd, cwd, env, shell,
+                           logo.log_stdout, logo.log_stderr))
+    loop.close()
+
+    if rc and fail:
+        msg = 'Command "{cwd}$ {cmd}" failed\n'.format(cwd=cwd, cmd=cmdstr)
+        for line in logo.stderr:
+            msg += line
+        logging.error(msg)
+        sys.exit(rc)
+
+    return (rc, ''.join(logo.stdout))
+
+
+def find_program(paths, name):
+    for path in paths.split(os.pathsep):
+        prg = os.path.join(path, name)
+        if os.path.isfile(prg):
+            return prg
+    return None
+
+
+def get_oe_environ(config, build_dir):
+    # nasty side effect function: running oe-init-build-env also
+    # creates the conf directory
+
+    oe_path = None
+    for repo in config.get_repos():
+        if os.path.exists(repo.path + '/oe-init-build-env'):
+            oe_path = repo.path
+            break
+    if not oe_path:
+        logging.error('Did not find oe-init-build-env')
+        sys.exit(1)
+
+    get_bb_env_file = tempfile.mktemp()
+    with open(get_bb_env_file, 'w') as f:
+        script = """#!/bin/bash
+        source oe-init-build-env $1 > /dev/null 2>&1
+        env
+        """
+        f.write(script)
+    os.chmod(get_bb_env_file, 0o775)
+
+    env = {}
+    env['PATH'] = '/bin:/usr/bin'
+
+    (rc, output) = run_cmd([get_bb_env_file, build_dir],
+                           cwd=oe_path, env=env, liveupdate=False)
+
+    os.remove(get_bb_env_file)
+
+    env = {}
+    for line in output.splitlines():
+        try:
+            (key, val) = line.split('=', 1)
+            env[key] = val
+        except:
+            pass
+
+    vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR']
+    if 'BB_ENV_EXTRAWHITE' in env:
+        ew = env['BB_ENV_EXTRAWHITE'] + ' '.join(vars)
+        env.update({'BB_ENV_EXTRAWHITE': ew})
+
+    for v in vars:
+        if v in os.environ:
+            env[v] = os.environ[v]
+
+    return env
+
+
+def ssh_add_key(env, key):
+    p = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None,
+              stderr=PIPE, env=env)
+    error = p.communicate(input=str.encode(key))[1]
+    if p.returncode and error:
+        logging.error('failed to add ssh key: {}'.format(error))
+
+
+def ssh_cleanup_agent(config):
+    """Removes the identities and stop the ssh-agent instance """
+    # remove the identities
+    p = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ)
+    p.wait()
+    if p.returncode != 0:
+        logging.error('failed to delete SSH identities')
+
+    # stop the ssh-agent
+    p = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ)
+    p.wait()
+    if p.returncode != 0:
+        logging.error('failed to stop SSH agent')
+
+
+def ssh_setup_agent(config, envkeys=['SSH_PRIVATE_KEY']):
+    output = os.popen('/usr/bin/ssh-agent -s').readlines()
+    for line in output:
+        matches = re.search("(\S+)\=(\S+)\;", line)
+        if matches:
+            config.environ[matches.group(1)] = matches.group(2)
+
+    for ek in envkeys:
+        key = os.environ.get(ek)
+        if key:
+            ssh_add_key(config.environ, key)
+        else:
+            logging.warning('{} is missing'.format(ek))
+
+
+def ssh_no_host_key_check(config):
+    home = os.path.expanduser('~')
+    if not os.path.exists(home + '/.ssh'):
+        os.mkdir(home + '/.ssh')
+    with open(home + '/.ssh/config', 'w') as f:
+        f.write('Host *\n\tStrictHostKeyChecking no\n\n')

kas/repos.py

@@ -0,0 +1,57 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+from urllib.parse import urlparse
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class Repo:
+    def __init__(self, url, path, refspec=None, sublayers=None):
+        self.url = url
+        self.path = path
+        self.refspec = refspec
+        self.sublayers = sublayers
+        self.name = os.path.basename(self.path)
+        self.git_operation_disabled = False
+
+    def disable_git_operations(self):
+        self.git_operation_disabled = True
+
+    def __getattr__(self, item):
+        if item == 'layers':
+            if not self.sublayers:
+                return [self.path]
+            else:
+                return [self.path + '/' + l for l in self.sublayers]
+        elif item == 'qualified_name':
+            url = urlparse(self.url)
+            return ('{url.netloc}{url.path}'.format(url=url)
+                                            .replace('@', '.')
+                                            .replace(':', '.')
+                                            .replace('/', '.')
+                                            .replace('*', '.'))
+
+    def __str__(self):
+        return '%s:%s %s' % (self.url, self.refspec, self.sublayers)

kas/shell.py

@@ -0,0 +1,81 @@
+# kas - setup tool for bitbake based projects
+#
+# Copyright (c) Siemens AG, 2017
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import subprocess
+from kas.config import load_config
+from kas.libcmds import (Macro, Command, SetupProxy, SetupEnviron, SetupHome)
+
+__license__ = 'MIT'
+__copyright__ = 'Copyright (c) Siemens AG, 2017'
+
+
+class Shell:
+    def __init__(self, parser):
+        sh_prs = parser.add_parser('shell')
+
+        sh_prs.add_argument('config',
+                            help='Config file')
+        sh_prs.add_argument('--target',
+                            help='Select target to build',
+                            default='core-image-minimal')
+        sh_prs.add_argument('--skip',
+                            help='Skip build steps',
+                            default=[])
+        sh_prs.add_argument('-c', '--command',
+                            help='Run command',
+                            default='')
+
+    def run(self, args):
+        if args.cmd != 'shell':
+            return False
+
+        cfg = load_config(args.config, args.target)
+
+        macro = Macro()
+
+        macro.add(SetupProxy())
+        macro.add(SetupEnviron())
+        macro.add(SetupHome())
+        macro.add(ShellCommand(args.command))
+
+        macro.run(cfg, args.skip)
+
+        return True
+
+
+class ShellCommand(Command):
+    def __init__(self, cmd):
+        Command.__init__(self)
+        self.cmd = []
+        if cmd:
+            self.cmd = cmd
+
+    def __str__(self):
+        return 'shell'
+
+    def execute(self, config):
+        cmd = ['/bin/sh']
+        if self.cmd:
+            cmd.append('-c')
+            cmd.append(self.cmd)
+        subprocess.call(cmd, env=config.environ,
+                        cwd=config.build_dir)