SiegfriedSiegert/PTU5KAS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

run pylint3 and fixed report

Browse Source 
This commit adds a pylint configuration and fixed all found issues.

Signed-off-by: Claudius Heine <ch@denx.de>
This commit is contained in:
Claudius Heine
2017-06-21 13:32:56 +02:00
committed by Daniel Wagner
parent 6bc8e08459
commit 33a21c8d0d
13 changed files with 917 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
kas/__init__.py
View File

@@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
kas - setup tool for bitbake based projects
"""
from .__version__ import __version__

3
kas/__main__.py
View File

@@ -21,6 +21,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
The main entry point of kas, a setup tool for bitbake based projects
"""
from .kas import main

27
kas/__version__.py
View File

@@ -1 +1,28 @@
# kas - setup tool for bitbake based projects
#
# Copyright (c) Siemens AG, 2017
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains the version of kas.
"""
__license__ = 'MIT'
__copyright__ = 'Copyright (c) Siemens AG, 2017'
__version__ = '0.9.0'

21
kas/build.py
View File

@@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
The build plugin for kas.
"""
import os
from .config import load_config
@@ -33,6 +36,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Build:
"""
This class implements the build plugin for kas.
"""
def __init__(self, parser):
bld_psr = parser.add_parser('build')
@@ -49,6 +56,11 @@ class Build:
default=[])
def run(self, args):
"""
Executes the build command of the kas plugin.
"""
# pylint: disable=no-self-use
if args.cmd != 'build':
return False
@@ -82,14 +94,21 @@ class Build:
class BuildCommand(Command):
"""
Implement the bitbake build step.
"""
def __init__(self, task):
Command.__init__
super().__init__()
self.task = task
def __str__(self):
return 'build'
def execute(self, config):
"""
Executes the bitbake build command.
"""
# Start bitbake build of image
bitbake = find_program(config.environ['PATH'], 'bitbake')
run_cmd([bitbake, '-k', config.get_bitbake_target(), '-c', self.task],

229
kas/config.py
View File

@@ -19,14 +19,31 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains the implementation of the kas configuration.
"""
import os
import sys
import logging
import errno
import json
import platform
import yaml
try:
from distro import id as get_distro_id
except ImportError:
import platform
def get_distro_id():
"""
Wrapper around platform.dist to simulate distro.id
platform.dist is deprecated and will be removed in python 3.7
Use the 'distro' package instead.
"""
# pylint: disable=deprecated-method
return platform.dist()[0]
from .repos import Repo
from .libkas import run_cmd
@@ -35,24 +52,39 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Config:
"""
This is an abstract class, that defines the interface of the
kas configuration.
"""
def __init__(self):
self.__kas_work_dir = os.environ.get('KAS_WORK_DIR', os.getcwd())
self.environ = {}
@property
def build_dir(self):
"""
The path of the build directory.
"""
return os.path.join(self.__kas_work_dir, 'build')
@property
def kas_work_dir(self):
"""
The path to the kas work directory.
"""
return self.__kas_work_dir
def setup_environ(self):
(distro, version, id) = platform.dist()
if distro in ['fedora', 'SuSE']:
"""
Sets the environment variables for process that are
started by kas.
"""
distro_id = get_distro_id()
if distro_id in ['fedora', 'SuSE']:
self.environ = {'LC_ALL': 'en_US.utf8',
'LANG': 'en_US.utf8',
'LANGUAGE': 'en_US'}
elif distro in ['Ubuntu', 'debian']:
elif distro_id in ['Ubuntu', 'debian']:
self.environ = {'LC_ALL': 'en_US.UTF-8',
'LANG': 'en_US.UTF-8',
'LANGUAGE': 'en_US:en'}
@@ -61,17 +93,27 @@ class Config:
self.environ = {}
def get_repo_ref_dir(self):
"""
The path to the directory that contains the repository references.
"""
# pylint: disable=no-self-use
return os.environ.get('KAS_REPO_REF_DIR', None)
class ConfigPython(Config):
"""
Implementation of a configuration that uses a Python script.
"""
def __init__(self, filename, target):
# pylint: disable=exec-used
super().__init__()
self.filename = os.path.abspath(filename)
try:
with open(self.filename) as file:
with open(self.filename) as fds:
env = {}
data = file.read()
data = fds.read()
exec(data, env)
self._config = env
except IOError:
@@ -82,80 +124,119 @@ class ConfigPython(Config):
self.setup_environ()
def __str__(self):
s = 'target: {}\n'.format(self.target)
s += 'repos:\n'
for r in self.get_repos():
s += ' {}\n'.format(r.__str__())
s += 'environ:\n'
for k, v in self.environ.items():
s += ' {} = {}\n'.format(k, v)
s += 'proxy:\n'
for k, v in self.get_proxy_config().items():
s += ' {} = {}\n'.format(k, v)
return s
output = 'target: {}\n'.format(self.target)
output += 'repos:\n'
for repo in self.get_repos():
output += ' {}\n'.format(repo.__str__())
output += 'environ:\n'
for key, value in self.environ.items():
output += ' {} = {}\n'.format(key, value)
output += 'proxy:\n'
for key, value in self.get_proxy_config().items():
output += ' {} = {}\n'.format(key, value)
return output
def pre_hook(self, fname):
"""
Returns a function that is executed before every command or None.
"""
try:
self._config[fname + '_prepend'](self)
except KeyError:
pass
def post_hook(self, fname):
"""
Returs a function that is executed after every command or None.
"""
try:
self._config[fname + '_append'](self)
except KeyError:
pass
def get_hook(self, fname):
"""
Returns a function that is executed instead of the command or None.
"""
try:
return self._config[fname]
except KeyError:
return None
def create_config(self, target):
"""
Sets the configuration for `target`
"""
self.target = target
self.repos = self._config['get_repos'](self, target)
def get_proxy_config(self):
"""
Returns the proxy settings
"""
return self._config['get_proxy_config']()
def get_repos(self):
"""
Returns the list of repos
"""
return iter(self.repos)
def get_target(self):
"""
Returns the target
"""
return self.target
def get_bitbake_target(self):
"""
Return the bitbake target
"""
try:
return self._config['get_bitbake_target'](self)
except KeyError:
return self.target
def get_bblayers_conf_header(self):
"""
Returns the bblayers.conf header
"""
try:
return self._config['get_bblayers_conf_header']()
except KeyError:
return ''
def get_local_conf_header(self):
"""
Returns the local.conf header
"""
try:
return self._config['get_local_conf_header']()
except:
except KeyError:
return ''
def get_machine(self):
"""
Returns the machine
"""
try:
return self._config['get_machine'](self)
except KeyError:
return 'qemu'
def get_distro(self):
"""
Returns the distro
"""
try:
return self._config['get_distro'](self)
except KeyError:
return 'poky'
def get_gitlabci_config(self):
"""
Returns the GitlabCI configuration
"""
try:
return self._config['get_gitlabci_config'](self)
except KeyError:
@@ -163,21 +244,37 @@ class ConfigPython(Config):
class ConfigStatic(Config):
def __init__(self, filename, target):
"""
An abstract class for static configuration files
"""
def __init__(self, filename, _):
super().__init__()
self.filename = os.path.abspath(filename)
self._config = []
self._config = {}
def pre_hook(self, target):
def pre_hook(self, _):
"""
Not used
"""
pass
def post_hook(self, target):
def post_hook(self, _):
"""
Not used
"""
pass
def get_hook(self, fname):
return None
def get_hook(self, _):
"""
Not used
"""
pass
def get_proxy_config(self):
"""
Returns the proxy settings
"""
try:
return self._config['proxy_config']
except KeyError:
@@ -186,6 +283,9 @@ class ConfigStatic(Config):
'no_proxy': os.environ.get('no_proxy', '')}
def get_repos(self):
"""
Returns the list of repos
"""
repos = []
for repo in self._config['repos']:
try:
@@ -196,57 +296,75 @@ class ConfigStatic(Config):
url = repo['url']
if url == '':
# in-tree configuration
(rc, output) = run_cmd(['/usr/bin/git',
'rev-parse',
'--show-toplevel'],
cwd=os.path.dirname(self.filename),
env=self.environ)
(_, output) = run_cmd(['/usr/bin/git',
'rev-parse',
'--show-toplevel'],
cwd=os.path.dirname(self.filename),
env=self.environ)
url = output.strip()
r = Repo(url=url,
path=url,
sublayers=sublayers)
r.disable_git_operations()
rep = Repo(url=url,
path=url,
sublayers=sublayers)
rep.disable_git_operations()
else:
name = os.path.basename(os.path.splitext(url)[0])
r = Repo(url=url,
path=os.path.join(self.kas_work_dir, name),
refspec=repo['refspec'],
sublayers=sublayers)
repos.append(r)
rep = Repo(url=url,
path=os.path.join(self.kas_work_dir, name),
refspec=repo['refspec'],
sublayers=sublayers)
repos.append(rep)
return repos
def get_bitbake_target(self):
"""
Return the bitbake target
"""
try:
return self._config['target']
except KeyError:
return 'core-image-minimal'
def get_bblayers_conf_header(self):
"""
Returns the bblayers.conf header
"""
try:
return self._config['bblayers_conf_header']
except KeyError:
return ''
def get_local_conf_header(self):
"""
Returns the local.conf header
"""
try:
return self._config['local_conf_header']
except KeyError:
return ''
def get_machine(self):
"""
Returns the machine
"""
try:
return self._config['machine']
except KeyError:
return 'qemu'
def get_distro(self):
"""
Returns the distro
"""
try:
return self._config['distro']
except KeyError:
return 'poky'
def get_gitlabci_config(self):
"""
Returns the GitlabCI configuration
"""
try:
return self._config['gitlabci_config']
except KeyError:
@@ -254,47 +372,60 @@ class ConfigStatic(Config):
class ConfigJson(ConfigStatic):
"""
Implements the configuration based on JSON files
"""
def __init__(self, filename, target):
super().__init__(filename, target)
self.filename = os.path.abspath(filename)
try:
with open(self.filename, 'r') as f:
self._config = json.load(f)
with open(self.filename, 'r') as fds:
self._config = json.load(fds)
except json.decoder.JSONDecodeError as msg:
logging.error('Could not load JSON config: {}'.format(msg))
logging.error('Could not load JSON config: %s', msg)
sys.exit(1)
self.setup_environ()
def get_bblayers_conf_header(self):
list = super().get_bblayers_conf_header()
header_list = super().get_bblayers_conf_header()
conf = ''
for line in list:
for line in header_list:
conf += str(line) + '\n'
return conf
def get_local_conf_header(self):
list = super().get_local_conf_header()
header_list = super().get_local_conf_header()
conf = ''
for line in list:
for line in header_list:
conf += str(line) + '\n'
return conf
class ConfigYaml(ConfigStatic):
"""
Implements the configuration based on Yaml files
"""
def __init__(self, filename, target):
super().__init__(filename, target)
self.filename = os.path.abspath(filename)
try:
with open(self.filename, 'r') as f:
self._config = yaml.load(f)
with open(self.filename, 'r') as fds:
self._config = yaml.load(fds)
except yaml.loader.ParserError as msg:
logging.error('Could not load YAML config: {}'.format(msg))
logging.error('Could not load YAML config: %s', msg)
sys.exit(1)
self.setup_environ()
def load_config(filename, target):
f, ext = os.path.splitext(filename)
"""
Return configuration generated from `filename`.
"""
# pylint: disable=redefined-variable-type
(_, ext) = os.path.splitext(filename)
if ext == '.py':
cfg = ConfigPython(filename, target)
elif ext == '.json':

38
kas/kas.py
View File

@@ -21,6 +21,10 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module is the main entry point for kas, setup tool for bitbake based
projects
"""
import argparse
import traceback
@@ -31,9 +35,9 @@ import pkg_resources
try:
import colorlog
have_colorlog = True
HAVE_COLORLOG = True
except ImportError:
have_colorlog = False
HAVE_COLORLOG = False
from .build import Build
from .shell import Shell
@@ -44,27 +48,34 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
def create_logger():
"""
Setup the logging environment
"""
log = logging.getLogger() # root logger
log.setLevel(logging.INFO)
format = '%(asctime)s - %(levelname)-8s - %(message)s'
format_str = '%(asctime)s - %(levelname)-8s - %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
if have_colorlog and os.isatty(2):
cformat = '%(log_color)s' + format
if HAVE_COLORLOG and os.isatty(2):
cformat = '%(log_color)s' + format_str
colors = {'DEBUG': 'reset',
'INFO': 'reset',
'WARNING': 'bold_yellow',
'ERROR': 'bold_red',
'CRITICAL': 'bold_red'}
f = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
formatter = colorlog.ColoredFormatter(cformat, date_format,
log_colors=colors)
else:
f = logging.Formatter(format, date_format)
ch = logging.StreamHandler()
ch.setFormatter(f)
log.addHandler(ch)
formatter = logging.Formatter(format_str, date_format)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
log.addHandler(stream_handler)
return logging.getLogger(__name__)
def kas(argv):
"""
The main entry point of kas.
"""
create_logger()
parser = argparse.ArgumentParser(description='Steer ebs-yocto builds')
@@ -96,10 +107,15 @@ def kas(argv):
def main():
"""
The main function that operates as a wrapper around kas.
"""
# pylint: disable=broad-except
try:
sys.exit(kas(sys.argv[1:]))
except Exception as err:
logging.error('%s' % err)
logging.error('%s', err)
traceback.print_exc()
sys.exit(1)

179
kas/libcmds.py
View File

@@ -19,12 +19,14 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contain common commands used by kas plugins.
"""
import tempfile
import logging
import shutil
import os
from urllib.parse import urlparse
from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check,
run_cmd, get_oe_environ)
@@ -33,41 +35,63 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Macro:
"""
Contains commands and provide method to run them.
"""
def __init__(self):
self.commands = []
def add(self, command):
"""
Appends commands to the command list.
"""
self.commands.append(command)
def run(self, config, skip=[]):
for c in self.commands:
name = str(c)
if name in skip:
def run(self, config, skip=None):
"""
Runs command from the command list respective to the configuration.
"""
skip = skip or []
for command in self.commands:
command_name = str(command)
if command_name in skip:
continue
pre = config.pre_hook(name)
if pre:
logging.debug('execute ' + pre)
pre(config)
cmd = config.get_hook(name)
if cmd:
logging.debug('execute ' + cmd)
cmd(config)
pre_hook = config.pre_hook(command_name)
if pre_hook:
logging.debug('execute %s', pre_hook)
pre_hook(config)
command_hook = config.get_hook(command_name)
if command_hook:
logging.debug('execute %s', command_hook)
command_hook(config)
else:
logging.debug('execute ' + str(c))
c.execute(config)
post = config.post_hook(name)
if post:
logging.debug('execute ' + post)
post(config)
logging.debug('execute %s', command_name)
command.execute(config)
post_hook = config.post_hook(command_name)
if post_hook:
logging.debug('execute %s', post_hook)
post_hook(config)
class Command:
"""
An abstract class that defines the interface of a command.
"""
def execute(self, config):
"""
This method executes the command.
"""
pass
class SetupHome(Command):
"""
Setups the home directory of kas.
"""
def __init__(self):
super().__init__()
self.tmpdirname = tempfile.mkdtemp()
def __del__(self):
@@ -77,14 +101,18 @@ class SetupHome(Command):
return 'setup_home'
def execute(self, config):
with open(self.tmpdirname + '/.wgetrc', 'w') as f:
f.write('\n')
with open(self.tmpdirname + '/.netrc', 'w') as f:
f.write('\n')
with open(self.tmpdirname + '/.wgetrc', 'w') as fds:
fds.write('\n')
with open(self.tmpdirname + '/.netrc', 'w') as fds:
fds.write('\n')
config.environ['HOME'] = self.tmpdirname
class SetupDir(Command):
"""
Creates the build directory.
"""
def __str__(self):
return 'setup_dir'
@@ -95,6 +123,10 @@ class SetupDir(Command):
class SetupSSHAgent(Command):
"""
Setup the ssh agent configuration.
"""
def __str__(self):
return 'setup_ssh_agent'
@@ -104,7 +136,9 @@ class SetupSSHAgent(Command):
class CleanupSSHAgent(Command):
"""Remove all the identities and stop the ssh-agent instance"""
"""
Remove all the identities and stop the ssh-agent instance.
"""
def __str__(self):
return 'cleanup_ssh_agent'
@@ -114,6 +148,10 @@ class CleanupSSHAgent(Command):
class SetupProxy(Command):
"""
Setups proxy configuration in the kas environment.
"""
def __str__(self):
return 'setup_proxy'
@@ -122,6 +160,10 @@ class SetupProxy(Command):
class SetupEnviron(Command):
"""
Setups the kas environment.
"""
def __str__(self):
return 'setup_environ'
@@ -130,34 +172,39 @@ class SetupEnviron(Command):
class WriteConfig(Command):
"""
Writes bitbake configuration files into the build directory.
"""
def __str__(self):
return 'write_config'
def execute(self, config):
self._write_bblayers_conf(config)
self._write_local_conf(config)
def _write_bblayers_conf(config):
filename = config.build_dir + '/conf/bblayers.conf'
with open(filename, 'w') as fds:
fds.write(config.get_bblayers_conf_header())
fds.write('BBLAYERS ?= " \\\n')
for repo in config.get_repos():
fds.write(' \\\n'.join(repo.layers + ['']))
fds.write('"\n')
def _append_layers(self, config, file):
for repo in config.get_repos():
file.write(' \\\n'.join(repo.layers + ['']))
def _write_local_conf(config):
filename = config.build_dir + '/conf/local.conf'
with open(filename, 'w') as fds:
fds.write(config.get_local_conf_header())
fds.write('MACHINE ?= "{}"\n'.format(config.get_machine()))
fds.write('DISTRO ?= "{}"\n'.format(config.get_distro()))
def _write_bblayers_conf(self, config):
filename = config.build_dir + '/conf/bblayers.conf'
with open(filename, 'w') as file:
file.write(config.get_bblayers_conf_header())
file.write('BBLAYERS ?= " \\\n')
self._append_layers(config, file)
file.write('"\n')
def _write_local_conf(self, config):
filename = config.build_dir + '/conf/local.conf'
with open(filename, 'w') as file:
file.write(config.get_local_conf_header())
file.write('MACHINE ?= "{}"\n'.format(config.get_machine()))
file.write('DISTRO ?= "{}"\n'.format(config.get_distro()))
_write_bblayers_conf(config)
_write_local_conf(config)
class ReposFetch(Command):
"""
Fetches repositories defined in the configuration
"""
def __str__(self):
return 'repos_fetch'
@@ -170,8 +217,7 @@ class ReposFetch(Command):
os.makedirs(os.path.dirname(repo.path), exist_ok=True)
gitsrcdir = os.path.join(config.get_repo_ref_dir() or '',
repo.qualified_name)
logging.debug('Looking for repo ref dir in {}'.
format(gitsrcdir))
logging.debug('Looking for repo ref dir in %s', gitsrcdir)
if config.get_repo_ref_dir() and os.path.exists(gitsrcdir):
run_cmd(['/usr/bin/git',
'clone',
@@ -187,22 +233,26 @@ class ReposFetch(Command):
continue
# Does refspec in the current repository?
(rc, output) = run_cmd(['/usr/bin/git', 'cat-file',
'-t', repo.refspec], env=config.environ,
cwd=repo.path, fail=False)
if rc == 0:
(retc, output) = run_cmd(['/usr/bin/git', 'cat-file',
'-t', repo.refspec], env=config.environ,
cwd=repo.path, fail=False)
if retc == 0:
continue
# No it is missing, try to fetch
(rc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
env=config.environ,
cwd=repo.path, fail=False)
if rc:
logging.warning('Could not update repository {}: {}'.
format(repo.name, output))
(retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
env=config.environ,
cwd=repo.path, fail=False)
if retc:
logging.warning('Could not update repository %s: %s',
repo.name, output)
class ReposCheckout(Command):
"""
Ensures that the right revision of each repo is check out.
"""
def __str__(self):
return 'repos_checkout'
@@ -212,22 +262,21 @@ class ReposCheckout(Command):
continue
# Check if repos is dirty
(rc, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
env=config.environ, cwd=repo.path,
fail=False)
(_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
env=config.environ, cwd=repo.path,
fail=False)
if len(output):
logging.warning('Repo {} is dirty. no checkout'.
format(repo.name))
logging.warning('Repo %s is dirty. no checkout', repo.name)
continue
# Check if current HEAD is what in the config file is defined.
(rc, output) = run_cmd(['/usr/bin/git', 'rev-parse',
'--verify', 'HEAD'], env=config.environ,
cwd=repo.path)
(_, output) = run_cmd(['/usr/bin/git', 'rev-parse',
'--verify', 'HEAD'],
env=config.environ, cwd=repo.path)
if output.strip() == repo.refspec:
logging.info(('Repo {} has already checkout out correct '
'refspec. nothing to do').format(repo.name))
logging.info('Repo %s has already checkout out correct '
'refspec. nothing to do', repo.name)
continue
run_cmd(['/usr/bin/git', 'checkout', '-q',

138
kas/libkas.py
View File

@@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains the core implementation of kas.
"""
import re
import os
@@ -33,38 +36,58 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class LogOutput:
"""
Handles the log output of executed applications
"""
def __init__(self, live):
self.live = live
self.stdout = []
self.stderr = []
def log_stdout(self, line):
"""
This method is called when a line over stdout is received.
"""
if self.live:
logging.info(line.strip())
self.stdout.append(line)
def log_stderr(self, line):
"""
This method is called when a line over stderr is received.
"""
if self.live:
logging.error(line.strip())
self.stderr.append(line)
@asyncio.coroutine
def _read_stream(stream, cb):
def _read_stream(stream, callback):
"""
This asynchronious method reads from the output stream of the
application and transfers each line to the callback function.
"""
while True:
line = yield from stream.readline()
try:
line = line.decode('utf-8')
except:
logging.warning('Could not decode line from stream - ignore it')
except UnicodeDecodeError as err:
logging.warning('Could not decode line from stream, ignore it: %s',
err)
if line:
cb(line)
callback(line)
else:
break
@asyncio.coroutine
def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb):
"""
This function starts the subprocess, sets up the output stream
handlers and waits until the process has existed
"""
# pylint: disable=too-many-arguments
if shell:
process = yield from asyncio.create_subprocess_shell(
cmd,
@@ -89,14 +112,18 @@ def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb):
return ret
def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True):
rc = 0
stdout = []
stderr = []
def run_cmd(cmd, cwd, env=None, fail=True, shell=False, liveupdate=True):
"""
Starts a command.
"""
# pylint: disable=too-many-arguments
env = env or {}
retc = 0
cmdstr = cmd
if not shell:
cmdstr = ' '.join(cmd)
logging.info('{}$ {}'.format(cwd, cmdstr))
logging.info('%s$ %s', cwd, cmdstr)
logo = LogOutput(liveupdate)
if asyncio.get_event_loop().is_closed():
@@ -105,22 +132,25 @@ def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True):
else:
loop = asyncio.get_event_loop()
rc = loop.run_until_complete(
retc = loop.run_until_complete(
_stream_subprocess(cmd, cwd, env, shell,
logo.log_stdout, logo.log_stderr))
loop.close()
if rc and fail:
if retc and fail:
msg = 'Command "{cwd}$ {cmd}" failed\n'.format(cwd=cwd, cmd=cmdstr)
for line in logo.stderr:
msg += line
logging.error(msg)
sys.exit(rc)
sys.exit(retc)
return (rc, ''.join(logo.stdout))
return (retc, ''.join(logo.stdout))
def find_program(paths, name):
"""
Find a file within the paths array and returns its path.
"""
for path in paths.split(os.pathsep):
prg = os.path.join(path, name)
if os.path.isfile(prg):
@@ -129,6 +159,10 @@ def find_program(paths, name):
def get_oe_environ(config, build_dir):
"""
Create the openembedded environment variables.
"""
# pylint: disable=too-many-locals
# nasty side effect function: running oe-init-build-env also
# creates the conf directory
@@ -142,19 +176,19 @@ def get_oe_environ(config, build_dir):
sys.exit(1)
get_bb_env_file = tempfile.mktemp()
with open(get_bb_env_file, 'w') as f:
with open(get_bb_env_file, 'w') as fds:
script = """#!/bin/bash
source oe-init-build-env $1 > /dev/null 2>&1
env
"""
f.write(script)
fds.write(script)
os.chmod(get_bb_env_file, 0o775)
env = {}
env['PATH'] = '/bin:/usr/bin'
(rc, output) = run_cmd([get_bb_env_file, build_dir],
cwd=oe_path, env=env, liveupdate=False)
(_, output) = run_cmd([get_bb_env_file, build_dir],
cwd=oe_path, env=env, liveupdate=False)
os.remove(get_bb_env_file)
@@ -163,65 +197,77 @@ def get_oe_environ(config, build_dir):
try:
(key, val) = line.split('=', 1)
env[key] = val
except:
except ValueError:
pass
vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR']
env_vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR']
if 'BB_ENV_EXTRAWHITE' in env:
ew = env['BB_ENV_EXTRAWHITE'] + ' '.join(vars)
env.update({'BB_ENV_EXTRAWHITE': ew})
extra_white = env['BB_ENV_EXTRAWHITE'] + ' '.join(env_vars)
env.update({'BB_ENV_EXTRAWHITE': extra_white})
vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK',
'SHELL', 'TERM'])
env_vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK',
'SHELL', 'TERM'])
for v in vars:
if v in os.environ:
env[v] = os.environ[v]
for env_var in env_vars:
if env_var in os.environ:
env[env_var] = os.environ[env_var]
return env
def ssh_add_key(env, key):
p = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None,
stderr=PIPE, env=env)
error = p.communicate(input=str.encode(key))[1]
if p.returncode and error:
logging.error('failed to add ssh key: {}'.format(error))
"""
Add ssh key to the ssh-agent
"""
process = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None,
stderr=PIPE, env=env)
(_, error) = process.communicate(input=str.encode(key))
if process.returncode and error:
logging.error('failed to add ssh key: %s', error)
def ssh_cleanup_agent(config):
"""Removes the identities and stop the ssh-agent instance """
"""
Removes the identities and stop the ssh-agent instance
"""
# remove the identities
p = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ)
p.wait()
if p.returncode != 0:
process = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ)
process.wait()
if process.returncode != 0:
logging.error('failed to delete SSH identities')
# stop the ssh-agent
p = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ)
p.wait()
if p.returncode != 0:
process = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ)
process.wait()
if process.returncode != 0:
logging.error('failed to stop SSH agent')
def ssh_setup_agent(config, envkeys=['SSH_PRIVATE_KEY']):
def ssh_setup_agent(config, envkeys=None):
"""
Starts the ssh-agent
"""
envkeys = envkeys or ['SSH_PRIVATE_KEY']
output = os.popen('/usr/bin/ssh-agent -s').readlines()
for line in output:
matches = re.search("(\S+)\=(\S+)\;", line)
matches = re.search(r"(\S+)\=(\S+)\;", line)
if matches:
config.environ[matches.group(1)] = matches.group(2)
for ek in envkeys:
key = os.environ.get(ek)
for envkey in envkeys:
key = os.environ.get(envkey)
if key:
ssh_add_key(config.environ, key)
else:
logging.warning('{} is missing'.format(ek))
logging.warning('%s is missing', envkey)
def ssh_no_host_key_check(config):
def ssh_no_host_key_check(_):
"""
Disables ssh host key check
"""
home = os.path.expanduser('~')
if not os.path.exists(home + '/.ssh'):
os.mkdir(home + '/.ssh')
with open(home + '/.ssh/config', 'w') as f:
f.write('Host *\n\tStrictHostKeyChecking no\n\n')
with open(home + '/.ssh/config', 'w') as fds:
fds.write('Host *\n\tStrictHostKeyChecking no\n\n')

21
kas/repos.py
View File

@@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains the Repo class.
"""
import os
from urllib.parse import urlparse
@@ -28,6 +31,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Repo:
"""
Represents a repository in the kas configuration.
"""
def __init__(self, url, path, refspec=None, sublayers=None):
self.url = url
self.path = path
@@ -37,6 +44,9 @@ class Repo:
self.git_operation_disabled = False
def disable_git_operations(self):
"""
Disabled all git operation for this repository.
"""
self.git_operation_disabled = True
def __getattr__(self, item):
@@ -47,11 +57,12 @@ class Repo:
return [self.path + '/' + l for l in self.sublayers]
elif item == 'qualified_name':
url = urlparse(self.url)
return ('{url.netloc}{url.path}'.format(url=url)
.replace('@', '.')
.replace(':', '.')
.replace('/', '.')
.replace('*', '.'))
return ('{url.netloc}{url.path}'
.format(url=url)
.replace('@', '.')
.replace(':', '.')
.replace('/', '.')
.replace('*', '.'))
def __str__(self):
return '%s:%s %s' % (self.url, self.refspec, self.sublayers)

19
kas/shell.py
View File

@@ -19,6 +19,10 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains a kas plugin that opens a shell within the kas
environment
"""
import subprocess
from kas.config import load_config
@@ -29,6 +33,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Shell:
"""
Implements a kas plugin that opens a shell within the kas environment.
"""
def __init__(self, parser):
sh_prs = parser.add_parser('shell')
@@ -45,6 +53,11 @@ class Shell:
default='')
def run(self, args):
"""
Runs this kas plugin
"""
# pylint: disable= no-self-use
if args.cmd != 'shell':
return False
@@ -63,8 +76,12 @@ class Shell:
class ShellCommand(Command):
"""
This class implements the command that starts a shell.
"""
def __init__(self, cmd):
Command.__init__(self)
super().__init__()
self.cmd = []
if cmd:
self.cmd = cmd