run pylint3 and fixed report

This commit adds a pylint configuration and fixed all found issues.

Signed-off-by: Claudius Heine <ch@denx.de>
This commit is contained in:
Claudius Heine 2017-06-21 13:32:56 +02:00 committed by Daniel Wagner
parent 6bc8e08459
commit 33a21c8d0d
13 changed files with 917 additions and 184 deletions

407
.pylintrc Normal file
View File

@ -0,0 +1,407 @@
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
ignore-patterns=
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality. This option is deprecated
# and it will be removed in Pylint 2.0.
optimize-ast=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=too-few-public-methods,locally-disabled
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]". This option is deprecated
# and it will be removed in Pylint 2.0.
files-output=no
# Tells whether to display a full report or only the messages
reports=yes
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six.moves,future.builtins
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=100
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=yes
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
property-classes=abc.abstractproperty
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
[ELIF]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators=contextlib.contextmanager
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=optparse
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception

View File

@ -29,7 +29,8 @@ Dependencies & installation
This projects depends on This projects depends on
- Python 3 - Python 3
- PyYAML - distro Python 3 package
- PyYAML Python 3 package
If you need Python 2 support consider sending patches. The most If you need Python 2 support consider sending patches. The most
obvious place to start is to use the trollius package intead of obvious place to start is to use the trollius package intead of

View File

@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
kas - setup tool for bitbake based projects
"""
from .__version__ import __version__ from .__version__ import __version__

View File

@ -21,6 +21,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
The main entry point of kas, a setup tool for bitbake based projects
"""
from .kas import main from .kas import main

View File

@ -1 +1,28 @@
# kas - setup tool for bitbake based projects
#
# Copyright (c) Siemens AG, 2017
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
This module contains the version of kas.
"""
__license__ = 'MIT'
__copyright__ = 'Copyright (c) Siemens AG, 2017'
__version__ = '0.9.0' __version__ = '0.9.0'

View File

@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
The build plugin for kas.
"""
import os import os
from .config import load_config from .config import load_config
@ -33,6 +36,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Build: class Build:
"""
This class implements the build plugin for kas.
"""
def __init__(self, parser): def __init__(self, parser):
bld_psr = parser.add_parser('build') bld_psr = parser.add_parser('build')
@ -49,6 +56,11 @@ class Build:
default=[]) default=[])
def run(self, args): def run(self, args):
"""
Executes the build command of the kas plugin.
"""
# pylint: disable=no-self-use
if args.cmd != 'build': if args.cmd != 'build':
return False return False
@ -82,14 +94,21 @@ class Build:
class BuildCommand(Command): class BuildCommand(Command):
"""
Implement the bitbake build step.
"""
def __init__(self, task): def __init__(self, task):
Command.__init__ super().__init__()
self.task = task self.task = task
def __str__(self): def __str__(self):
return 'build' return 'build'
def execute(self, config): def execute(self, config):
"""
Executes the bitbake build command.
"""
# Start bitbake build of image # Start bitbake build of image
bitbake = find_program(config.environ['PATH'], 'bitbake') bitbake = find_program(config.environ['PATH'], 'bitbake')
run_cmd([bitbake, '-k', config.get_bitbake_target(), '-c', self.task], run_cmd([bitbake, '-k', config.get_bitbake_target(), '-c', self.task],

View File

@ -19,14 +19,31 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module contains the implementation of the kas configuration.
"""
import os import os
import sys import sys
import logging import logging
import errno import errno
import json import json
import platform
import yaml import yaml
try:
from distro import id as get_distro_id
except ImportError:
import platform
def get_distro_id():
"""
Wrapper around platform.dist to simulate distro.id
platform.dist is deprecated and will be removed in python 3.7
Use the 'distro' package instead.
"""
# pylint: disable=deprecated-method
return platform.dist()[0]
from .repos import Repo from .repos import Repo
from .libkas import run_cmd from .libkas import run_cmd
@ -35,24 +52,39 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Config: class Config:
"""
This is an abstract class, that defines the interface of the
kas configuration.
"""
def __init__(self): def __init__(self):
self.__kas_work_dir = os.environ.get('KAS_WORK_DIR', os.getcwd()) self.__kas_work_dir = os.environ.get('KAS_WORK_DIR', os.getcwd())
self.environ = {}
@property @property
def build_dir(self): def build_dir(self):
"""
The path of the build directory.
"""
return os.path.join(self.__kas_work_dir, 'build') return os.path.join(self.__kas_work_dir, 'build')
@property @property
def kas_work_dir(self): def kas_work_dir(self):
"""
The path to the kas work directory.
"""
return self.__kas_work_dir return self.__kas_work_dir
def setup_environ(self): def setup_environ(self):
(distro, version, id) = platform.dist() """
if distro in ['fedora', 'SuSE']: Sets the environment variables for process that are
started by kas.
"""
distro_id = get_distro_id()
if distro_id in ['fedora', 'SuSE']:
self.environ = {'LC_ALL': 'en_US.utf8', self.environ = {'LC_ALL': 'en_US.utf8',
'LANG': 'en_US.utf8', 'LANG': 'en_US.utf8',
'LANGUAGE': 'en_US'} 'LANGUAGE': 'en_US'}
elif distro in ['Ubuntu', 'debian']: elif distro_id in ['Ubuntu', 'debian']:
self.environ = {'LC_ALL': 'en_US.UTF-8', self.environ = {'LC_ALL': 'en_US.UTF-8',
'LANG': 'en_US.UTF-8', 'LANG': 'en_US.UTF-8',
'LANGUAGE': 'en_US:en'} 'LANGUAGE': 'en_US:en'}
@ -61,17 +93,27 @@ class Config:
self.environ = {} self.environ = {}
def get_repo_ref_dir(self): def get_repo_ref_dir(self):
"""
The path to the directory that contains the repository references.
"""
# pylint: disable=no-self-use
return os.environ.get('KAS_REPO_REF_DIR', None) return os.environ.get('KAS_REPO_REF_DIR', None)
class ConfigPython(Config): class ConfigPython(Config):
"""
Implementation of a configuration that uses a Python script.
"""
def __init__(self, filename, target): def __init__(self, filename, target):
# pylint: disable=exec-used
super().__init__() super().__init__()
self.filename = os.path.abspath(filename) self.filename = os.path.abspath(filename)
try: try:
with open(self.filename) as file: with open(self.filename) as fds:
env = {} env = {}
data = file.read() data = fds.read()
exec(data, env) exec(data, env)
self._config = env self._config = env
except IOError: except IOError:
@ -82,80 +124,119 @@ class ConfigPython(Config):
self.setup_environ() self.setup_environ()
def __str__(self): def __str__(self):
s = 'target: {}\n'.format(self.target) output = 'target: {}\n'.format(self.target)
s += 'repos:\n' output += 'repos:\n'
for r in self.get_repos(): for repo in self.get_repos():
s += ' {}\n'.format(r.__str__()) output += ' {}\n'.format(repo.__str__())
s += 'environ:\n' output += 'environ:\n'
for k, v in self.environ.items(): for key, value in self.environ.items():
s += ' {} = {}\n'.format(k, v) output += ' {} = {}\n'.format(key, value)
s += 'proxy:\n' output += 'proxy:\n'
for k, v in self.get_proxy_config().items(): for key, value in self.get_proxy_config().items():
s += ' {} = {}\n'.format(k, v) output += ' {} = {}\n'.format(key, value)
return s return output
def pre_hook(self, fname): def pre_hook(self, fname):
"""
Returns a function that is executed before every command or None.
"""
try: try:
self._config[fname + '_prepend'](self) self._config[fname + '_prepend'](self)
except KeyError: except KeyError:
pass pass
def post_hook(self, fname): def post_hook(self, fname):
"""
Returs a function that is executed after every command or None.
"""
try: try:
self._config[fname + '_append'](self) self._config[fname + '_append'](self)
except KeyError: except KeyError:
pass pass
def get_hook(self, fname): def get_hook(self, fname):
"""
Returns a function that is executed instead of the command or None.
"""
try: try:
return self._config[fname] return self._config[fname]
except KeyError: except KeyError:
return None return None
def create_config(self, target): def create_config(self, target):
"""
Sets the configuration for `target`
"""
self.target = target self.target = target
self.repos = self._config['get_repos'](self, target) self.repos = self._config['get_repos'](self, target)
def get_proxy_config(self): def get_proxy_config(self):
"""
Returns the proxy settings
"""
return self._config['get_proxy_config']() return self._config['get_proxy_config']()
def get_repos(self): def get_repos(self):
"""
Returns the list of repos
"""
return iter(self.repos) return iter(self.repos)
def get_target(self): def get_target(self):
"""
Returns the target
"""
return self.target return self.target
def get_bitbake_target(self): def get_bitbake_target(self):
"""
Return the bitbake target
"""
try: try:
return self._config['get_bitbake_target'](self) return self._config['get_bitbake_target'](self)
except KeyError: except KeyError:
return self.target return self.target
def get_bblayers_conf_header(self): def get_bblayers_conf_header(self):
"""
Returns the bblayers.conf header
"""
try: try:
return self._config['get_bblayers_conf_header']() return self._config['get_bblayers_conf_header']()
except KeyError: except KeyError:
return '' return ''
def get_local_conf_header(self): def get_local_conf_header(self):
"""
Returns the local.conf header
"""
try: try:
return self._config['get_local_conf_header']() return self._config['get_local_conf_header']()
except: except KeyError:
return '' return ''
def get_machine(self): def get_machine(self):
"""
Returns the machine
"""
try: try:
return self._config['get_machine'](self) return self._config['get_machine'](self)
except KeyError: except KeyError:
return 'qemu' return 'qemu'
def get_distro(self): def get_distro(self):
"""
Returns the distro
"""
try: try:
return self._config['get_distro'](self) return self._config['get_distro'](self)
except KeyError: except KeyError:
return 'poky' return 'poky'
def get_gitlabci_config(self): def get_gitlabci_config(self):
"""
Returns the GitlabCI configuration
"""
try: try:
return self._config['get_gitlabci_config'](self) return self._config['get_gitlabci_config'](self)
except KeyError: except KeyError:
@ -163,21 +244,37 @@ class ConfigPython(Config):
class ConfigStatic(Config): class ConfigStatic(Config):
def __init__(self, filename, target): """
An abstract class for static configuration files
"""
def __init__(self, filename, _):
super().__init__() super().__init__()
self.filename = os.path.abspath(filename) self.filename = os.path.abspath(filename)
self._config = [] self._config = {}
def pre_hook(self, target): def pre_hook(self, _):
"""
Not used
"""
pass pass
def post_hook(self, target): def post_hook(self, _):
"""
Not used
"""
pass pass
def get_hook(self, fname): def get_hook(self, _):
return None """
Not used
"""
pass
def get_proxy_config(self): def get_proxy_config(self):
"""
Returns the proxy settings
"""
try: try:
return self._config['proxy_config'] return self._config['proxy_config']
except KeyError: except KeyError:
@ -186,6 +283,9 @@ class ConfigStatic(Config):
'no_proxy': os.environ.get('no_proxy', '')} 'no_proxy': os.environ.get('no_proxy', '')}
def get_repos(self): def get_repos(self):
"""
Returns the list of repos
"""
repos = [] repos = []
for repo in self._config['repos']: for repo in self._config['repos']:
try: try:
@ -196,57 +296,75 @@ class ConfigStatic(Config):
url = repo['url'] url = repo['url']
if url == '': if url == '':
# in-tree configuration # in-tree configuration
(rc, output) = run_cmd(['/usr/bin/git', (_, output) = run_cmd(['/usr/bin/git',
'rev-parse', 'rev-parse',
'--show-toplevel'], '--show-toplevel'],
cwd=os.path.dirname(self.filename), cwd=os.path.dirname(self.filename),
env=self.environ) env=self.environ)
url = output.strip() url = output.strip()
r = Repo(url=url, rep = Repo(url=url,
path=url, path=url,
sublayers=sublayers) sublayers=sublayers)
r.disable_git_operations() rep.disable_git_operations()
else: else:
name = os.path.basename(os.path.splitext(url)[0]) name = os.path.basename(os.path.splitext(url)[0])
r = Repo(url=url, rep = Repo(url=url,
path=os.path.join(self.kas_work_dir, name), path=os.path.join(self.kas_work_dir, name),
refspec=repo['refspec'], refspec=repo['refspec'],
sublayers=sublayers) sublayers=sublayers)
repos.append(r) repos.append(rep)
return repos return repos
def get_bitbake_target(self): def get_bitbake_target(self):
"""
Return the bitbake target
"""
try: try:
return self._config['target'] return self._config['target']
except KeyError: except KeyError:
return 'core-image-minimal' return 'core-image-minimal'
def get_bblayers_conf_header(self): def get_bblayers_conf_header(self):
"""
Returns the bblayers.conf header
"""
try: try:
return self._config['bblayers_conf_header'] return self._config['bblayers_conf_header']
except KeyError: except KeyError:
return '' return ''
def get_local_conf_header(self): def get_local_conf_header(self):
"""
Returns the local.conf header
"""
try: try:
return self._config['local_conf_header'] return self._config['local_conf_header']
except KeyError: except KeyError:
return '' return ''
def get_machine(self): def get_machine(self):
"""
Returns the machine
"""
try: try:
return self._config['machine'] return self._config['machine']
except KeyError: except KeyError:
return 'qemu' return 'qemu'
def get_distro(self): def get_distro(self):
"""
Returns the distro
"""
try: try:
return self._config['distro'] return self._config['distro']
except KeyError: except KeyError:
return 'poky' return 'poky'
def get_gitlabci_config(self): def get_gitlabci_config(self):
"""
Returns the GitlabCI configuration
"""
try: try:
return self._config['gitlabci_config'] return self._config['gitlabci_config']
except KeyError: except KeyError:
@ -254,47 +372,60 @@ class ConfigStatic(Config):
class ConfigJson(ConfigStatic): class ConfigJson(ConfigStatic):
"""
Implements the configuration based on JSON files
"""
def __init__(self, filename, target): def __init__(self, filename, target):
super().__init__(filename, target) super().__init__(filename, target)
self.filename = os.path.abspath(filename) self.filename = os.path.abspath(filename)
try: try:
with open(self.filename, 'r') as f: with open(self.filename, 'r') as fds:
self._config = json.load(f) self._config = json.load(fds)
except json.decoder.JSONDecodeError as msg: except json.decoder.JSONDecodeError as msg:
logging.error('Could not load JSON config: {}'.format(msg)) logging.error('Could not load JSON config: %s', msg)
sys.exit(1) sys.exit(1)
self.setup_environ() self.setup_environ()
def get_bblayers_conf_header(self): def get_bblayers_conf_header(self):
list = super().get_bblayers_conf_header() header_list = super().get_bblayers_conf_header()
conf = '' conf = ''
for line in list: for line in header_list:
conf += str(line) + '\n' conf += str(line) + '\n'
return conf return conf
def get_local_conf_header(self): def get_local_conf_header(self):
list = super().get_local_conf_header() header_list = super().get_local_conf_header()
conf = '' conf = ''
for line in list: for line in header_list:
conf += str(line) + '\n' conf += str(line) + '\n'
return conf return conf
class ConfigYaml(ConfigStatic): class ConfigYaml(ConfigStatic):
"""
Implements the configuration based on Yaml files
"""
def __init__(self, filename, target): def __init__(self, filename, target):
super().__init__(filename, target) super().__init__(filename, target)
self.filename = os.path.abspath(filename) self.filename = os.path.abspath(filename)
try: try:
with open(self.filename, 'r') as f: with open(self.filename, 'r') as fds:
self._config = yaml.load(f) self._config = yaml.load(fds)
except yaml.loader.ParserError as msg: except yaml.loader.ParserError as msg:
logging.error('Could not load YAML config: {}'.format(msg)) logging.error('Could not load YAML config: %s', msg)
sys.exit(1) sys.exit(1)
self.setup_environ() self.setup_environ()
def load_config(filename, target): def load_config(filename, target):
f, ext = os.path.splitext(filename) """
Return configuration generated from `filename`.
"""
# pylint: disable=redefined-variable-type
(_, ext) = os.path.splitext(filename)
if ext == '.py': if ext == '.py':
cfg = ConfigPython(filename, target) cfg = ConfigPython(filename, target)
elif ext == '.json': elif ext == '.json':

View File

@ -21,6 +21,10 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module is the main entry point for kas, setup tool for bitbake based
projects
"""
import argparse import argparse
import traceback import traceback
@ -31,9 +35,9 @@ import pkg_resources
try: try:
import colorlog import colorlog
have_colorlog = True HAVE_COLORLOG = True
except ImportError: except ImportError:
have_colorlog = False HAVE_COLORLOG = False
from .build import Build from .build import Build
from .shell import Shell from .shell import Shell
@ -44,27 +48,34 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
def create_logger(): def create_logger():
"""
Setup the logging environment
"""
log = logging.getLogger() # root logger log = logging.getLogger() # root logger
log.setLevel(logging.INFO) log.setLevel(logging.INFO)
format = '%(asctime)s - %(levelname)-8s - %(message)s' format_str = '%(asctime)s - %(levelname)-8s - %(message)s'
date_format = '%Y-%m-%d %H:%M:%S' date_format = '%Y-%m-%d %H:%M:%S'
if have_colorlog and os.isatty(2): if HAVE_COLORLOG and os.isatty(2):
cformat = '%(log_color)s' + format cformat = '%(log_color)s' + format_str
colors = {'DEBUG': 'reset', colors = {'DEBUG': 'reset',
'INFO': 'reset', 'INFO': 'reset',
'WARNING': 'bold_yellow', 'WARNING': 'bold_yellow',
'ERROR': 'bold_red', 'ERROR': 'bold_red',
'CRITICAL': 'bold_red'} 'CRITICAL': 'bold_red'}
f = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors) formatter = colorlog.ColoredFormatter(cformat, date_format,
log_colors=colors)
else: else:
f = logging.Formatter(format, date_format) formatter = logging.Formatter(format_str, date_format)
ch = logging.StreamHandler() stream_handler = logging.StreamHandler()
ch.setFormatter(f) stream_handler.setFormatter(formatter)
log.addHandler(ch) log.addHandler(stream_handler)
return logging.getLogger(__name__) return logging.getLogger(__name__)
def kas(argv): def kas(argv):
"""
The main entry point of kas.
"""
create_logger() create_logger()
parser = argparse.ArgumentParser(description='Steer ebs-yocto builds') parser = argparse.ArgumentParser(description='Steer ebs-yocto builds')
@ -96,10 +107,15 @@ def kas(argv):
def main(): def main():
"""
The main function that operates as a wrapper around kas.
"""
# pylint: disable=broad-except
try: try:
sys.exit(kas(sys.argv[1:])) sys.exit(kas(sys.argv[1:]))
except Exception as err: except Exception as err:
logging.error('%s' % err) logging.error('%s', err)
traceback.print_exc() traceback.print_exc()
sys.exit(1) sys.exit(1)

View File

@ -19,12 +19,14 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module contain common commands used by kas plugins.
"""
import tempfile import tempfile
import logging import logging
import shutil import shutil
import os import os
from urllib.parse import urlparse
from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check, from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check,
run_cmd, get_oe_environ) run_cmd, get_oe_environ)
@ -33,41 +35,63 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Macro: class Macro:
"""
Contains commands and provide method to run them.
"""
def __init__(self): def __init__(self):
self.commands = [] self.commands = []
def add(self, command): def add(self, command):
"""
Appends commands to the command list.
"""
self.commands.append(command) self.commands.append(command)
def run(self, config, skip=[]): def run(self, config, skip=None):
for c in self.commands: """
name = str(c) Runs command from the command list respective to the configuration.
if name in skip: """
skip = skip or []
for command in self.commands:
command_name = str(command)
if command_name in skip:
continue continue
pre = config.pre_hook(name) pre_hook = config.pre_hook(command_name)
if pre: if pre_hook:
logging.debug('execute ' + pre) logging.debug('execute %s', pre_hook)
pre(config) pre_hook(config)
cmd = config.get_hook(name) command_hook = config.get_hook(command_name)
if cmd: if command_hook:
logging.debug('execute ' + cmd) logging.debug('execute %s', command_hook)
cmd(config) command_hook(config)
else: else:
logging.debug('execute ' + str(c)) logging.debug('execute %s', command_name)
c.execute(config) command.execute(config)
post = config.post_hook(name) post_hook = config.post_hook(command_name)
if post: if post_hook:
logging.debug('execute ' + post) logging.debug('execute %s', post_hook)
post(config) post_hook(config)
class Command: class Command:
"""
An abstract class that defines the interface of a command.
"""
def execute(self, config): def execute(self, config):
"""
This method executes the command.
"""
pass pass
class SetupHome(Command): class SetupHome(Command):
"""
Setups the home directory of kas.
"""
def __init__(self): def __init__(self):
super().__init__()
self.tmpdirname = tempfile.mkdtemp() self.tmpdirname = tempfile.mkdtemp()
def __del__(self): def __del__(self):
@ -77,14 +101,18 @@ class SetupHome(Command):
return 'setup_home' return 'setup_home'
def execute(self, config): def execute(self, config):
with open(self.tmpdirname + '/.wgetrc', 'w') as f: with open(self.tmpdirname + '/.wgetrc', 'w') as fds:
f.write('\n') fds.write('\n')
with open(self.tmpdirname + '/.netrc', 'w') as f: with open(self.tmpdirname + '/.netrc', 'w') as fds:
f.write('\n') fds.write('\n')
config.environ['HOME'] = self.tmpdirname config.environ['HOME'] = self.tmpdirname
class SetupDir(Command): class SetupDir(Command):
"""
Creates the build directory.
"""
def __str__(self): def __str__(self):
return 'setup_dir' return 'setup_dir'
@ -95,6 +123,10 @@ class SetupDir(Command):
class SetupSSHAgent(Command): class SetupSSHAgent(Command):
"""
Setup the ssh agent configuration.
"""
def __str__(self): def __str__(self):
return 'setup_ssh_agent' return 'setup_ssh_agent'
@ -104,7 +136,9 @@ class SetupSSHAgent(Command):
class CleanupSSHAgent(Command): class CleanupSSHAgent(Command):
"""Remove all the identities and stop the ssh-agent instance""" """
Remove all the identities and stop the ssh-agent instance.
"""
def __str__(self): def __str__(self):
return 'cleanup_ssh_agent' return 'cleanup_ssh_agent'
@ -114,6 +148,10 @@ class CleanupSSHAgent(Command):
class SetupProxy(Command): class SetupProxy(Command):
"""
Setups proxy configuration in the kas environment.
"""
def __str__(self): def __str__(self):
return 'setup_proxy' return 'setup_proxy'
@ -122,6 +160,10 @@ class SetupProxy(Command):
class SetupEnviron(Command): class SetupEnviron(Command):
"""
Setups the kas environment.
"""
def __str__(self): def __str__(self):
return 'setup_environ' return 'setup_environ'
@ -130,34 +172,39 @@ class SetupEnviron(Command):
class WriteConfig(Command): class WriteConfig(Command):
"""
Writes bitbake configuration files into the build directory.
"""
def __str__(self): def __str__(self):
return 'write_config' return 'write_config'
def execute(self, config): def execute(self, config):
self._write_bblayers_conf(config) def _write_bblayers_conf(config):
self._write_local_conf(config) filename = config.build_dir + '/conf/bblayers.conf'
with open(filename, 'w') as fds:
fds.write(config.get_bblayers_conf_header())
fds.write('BBLAYERS ?= " \\\n')
for repo in config.get_repos():
fds.write(' \\\n'.join(repo.layers + ['']))
fds.write('"\n')
def _append_layers(self, config, file): def _write_local_conf(config):
for repo in config.get_repos(): filename = config.build_dir + '/conf/local.conf'
file.write(' \\\n'.join(repo.layers + [''])) with open(filename, 'w') as fds:
fds.write(config.get_local_conf_header())
fds.write('MACHINE ?= "{}"\n'.format(config.get_machine()))
fds.write('DISTRO ?= "{}"\n'.format(config.get_distro()))
def _write_bblayers_conf(self, config): _write_bblayers_conf(config)
filename = config.build_dir + '/conf/bblayers.conf' _write_local_conf(config)
with open(filename, 'w') as file:
file.write(config.get_bblayers_conf_header())
file.write('BBLAYERS ?= " \\\n')
self._append_layers(config, file)
file.write('"\n')
def _write_local_conf(self, config):
filename = config.build_dir + '/conf/local.conf'
with open(filename, 'w') as file:
file.write(config.get_local_conf_header())
file.write('MACHINE ?= "{}"\n'.format(config.get_machine()))
file.write('DISTRO ?= "{}"\n'.format(config.get_distro()))
class ReposFetch(Command): class ReposFetch(Command):
"""
Fetches repositories defined in the configuration
"""
def __str__(self): def __str__(self):
return 'repos_fetch' return 'repos_fetch'
@ -170,8 +217,7 @@ class ReposFetch(Command):
os.makedirs(os.path.dirname(repo.path), exist_ok=True) os.makedirs(os.path.dirname(repo.path), exist_ok=True)
gitsrcdir = os.path.join(config.get_repo_ref_dir() or '', gitsrcdir = os.path.join(config.get_repo_ref_dir() or '',
repo.qualified_name) repo.qualified_name)
logging.debug('Looking for repo ref dir in {}'. logging.debug('Looking for repo ref dir in %s', gitsrcdir)
format(gitsrcdir))
if config.get_repo_ref_dir() and os.path.exists(gitsrcdir): if config.get_repo_ref_dir() and os.path.exists(gitsrcdir):
run_cmd(['/usr/bin/git', run_cmd(['/usr/bin/git',
'clone', 'clone',
@ -187,22 +233,26 @@ class ReposFetch(Command):
continue continue
# Does refspec in the current repository? # Does refspec in the current repository?
(rc, output) = run_cmd(['/usr/bin/git', 'cat-file', (retc, output) = run_cmd(['/usr/bin/git', 'cat-file',
'-t', repo.refspec], env=config.environ, '-t', repo.refspec], env=config.environ,
cwd=repo.path, fail=False) cwd=repo.path, fail=False)
if rc == 0: if retc == 0:
continue continue
# No it is missing, try to fetch # No it is missing, try to fetch
(rc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'], (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'],
env=config.environ, env=config.environ,
cwd=repo.path, fail=False) cwd=repo.path, fail=False)
if rc: if retc:
logging.warning('Could not update repository {}: {}'. logging.warning('Could not update repository %s: %s',
format(repo.name, output)) repo.name, output)
class ReposCheckout(Command): class ReposCheckout(Command):
"""
Ensures that the right revision of each repo is check out.
"""
def __str__(self): def __str__(self):
return 'repos_checkout' return 'repos_checkout'
@ -212,22 +262,21 @@ class ReposCheckout(Command):
continue continue
# Check if repos is dirty # Check if repos is dirty
(rc, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'], (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'],
env=config.environ, cwd=repo.path, env=config.environ, cwd=repo.path,
fail=False) fail=False)
if len(output): if len(output):
logging.warning('Repo {} is dirty. no checkout'. logging.warning('Repo %s is dirty. no checkout', repo.name)
format(repo.name))
continue continue
# Check if current HEAD is what in the config file is defined. # Check if current HEAD is what in the config file is defined.
(rc, output) = run_cmd(['/usr/bin/git', 'rev-parse', (_, output) = run_cmd(['/usr/bin/git', 'rev-parse',
'--verify', 'HEAD'], env=config.environ, '--verify', 'HEAD'],
cwd=repo.path) env=config.environ, cwd=repo.path)
if output.strip() == repo.refspec: if output.strip() == repo.refspec:
logging.info(('Repo {} has already checkout out correct ' logging.info('Repo %s has already checkout out correct '
'refspec. nothing to do').format(repo.name)) 'refspec. nothing to do', repo.name)
continue continue
run_cmd(['/usr/bin/git', 'checkout', '-q', run_cmd(['/usr/bin/git', 'checkout', '-q',

View File

@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module contains the core implementation of kas.
"""
import re import re
import os import os
@ -33,38 +36,58 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class LogOutput: class LogOutput:
"""
Handles the log output of executed applications
"""
def __init__(self, live): def __init__(self, live):
self.live = live self.live = live
self.stdout = [] self.stdout = []
self.stderr = [] self.stderr = []
def log_stdout(self, line): def log_stdout(self, line):
"""
This method is called when a line over stdout is received.
"""
if self.live: if self.live:
logging.info(line.strip()) logging.info(line.strip())
self.stdout.append(line) self.stdout.append(line)
def log_stderr(self, line): def log_stderr(self, line):
"""
This method is called when a line over stderr is received.
"""
if self.live: if self.live:
logging.error(line.strip()) logging.error(line.strip())
self.stderr.append(line) self.stderr.append(line)
@asyncio.coroutine @asyncio.coroutine
def _read_stream(stream, cb): def _read_stream(stream, callback):
"""
This asynchronious method reads from the output stream of the
application and transfers each line to the callback function.
"""
while True: while True:
line = yield from stream.readline() line = yield from stream.readline()
try: try:
line = line.decode('utf-8') line = line.decode('utf-8')
except: except UnicodeDecodeError as err:
logging.warning('Could not decode line from stream - ignore it') logging.warning('Could not decode line from stream, ignore it: %s',
err)
if line: if line:
cb(line) callback(line)
else: else:
break break
@asyncio.coroutine @asyncio.coroutine
def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb): def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb):
"""
This function starts the subprocess, sets up the output stream
handlers and waits until the process has existed
"""
# pylint: disable=too-many-arguments
if shell: if shell:
process = yield from asyncio.create_subprocess_shell( process = yield from asyncio.create_subprocess_shell(
cmd, cmd,
@ -89,14 +112,18 @@ def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb):
return ret return ret
def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True): def run_cmd(cmd, cwd, env=None, fail=True, shell=False, liveupdate=True):
rc = 0 """
stdout = [] Starts a command.
stderr = [] """
# pylint: disable=too-many-arguments
env = env or {}
retc = 0
cmdstr = cmd cmdstr = cmd
if not shell: if not shell:
cmdstr = ' '.join(cmd) cmdstr = ' '.join(cmd)
logging.info('{}$ {}'.format(cwd, cmdstr)) logging.info('%s$ %s', cwd, cmdstr)
logo = LogOutput(liveupdate) logo = LogOutput(liveupdate)
if asyncio.get_event_loop().is_closed(): if asyncio.get_event_loop().is_closed():
@ -105,22 +132,25 @@ def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True):
else: else:
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
rc = loop.run_until_complete( retc = loop.run_until_complete(
_stream_subprocess(cmd, cwd, env, shell, _stream_subprocess(cmd, cwd, env, shell,
logo.log_stdout, logo.log_stderr)) logo.log_stdout, logo.log_stderr))
loop.close() loop.close()
if rc and fail: if retc and fail:
msg = 'Command "{cwd}$ {cmd}" failed\n'.format(cwd=cwd, cmd=cmdstr) msg = 'Command "{cwd}$ {cmd}" failed\n'.format(cwd=cwd, cmd=cmdstr)
for line in logo.stderr: for line in logo.stderr:
msg += line msg += line
logging.error(msg) logging.error(msg)
sys.exit(rc) sys.exit(retc)
return (rc, ''.join(logo.stdout)) return (retc, ''.join(logo.stdout))
def find_program(paths, name): def find_program(paths, name):
"""
Find a file within the paths array and returns its path.
"""
for path in paths.split(os.pathsep): for path in paths.split(os.pathsep):
prg = os.path.join(path, name) prg = os.path.join(path, name)
if os.path.isfile(prg): if os.path.isfile(prg):
@ -129,6 +159,10 @@ def find_program(paths, name):
def get_oe_environ(config, build_dir): def get_oe_environ(config, build_dir):
"""
Create the openembedded environment variables.
"""
# pylint: disable=too-many-locals
# nasty side effect function: running oe-init-build-env also # nasty side effect function: running oe-init-build-env also
# creates the conf directory # creates the conf directory
@ -142,19 +176,19 @@ def get_oe_environ(config, build_dir):
sys.exit(1) sys.exit(1)
get_bb_env_file = tempfile.mktemp() get_bb_env_file = tempfile.mktemp()
with open(get_bb_env_file, 'w') as f: with open(get_bb_env_file, 'w') as fds:
script = """#!/bin/bash script = """#!/bin/bash
source oe-init-build-env $1 > /dev/null 2>&1 source oe-init-build-env $1 > /dev/null 2>&1
env env
""" """
f.write(script) fds.write(script)
os.chmod(get_bb_env_file, 0o775) os.chmod(get_bb_env_file, 0o775)
env = {} env = {}
env['PATH'] = '/bin:/usr/bin' env['PATH'] = '/bin:/usr/bin'
(rc, output) = run_cmd([get_bb_env_file, build_dir], (_, output) = run_cmd([get_bb_env_file, build_dir],
cwd=oe_path, env=env, liveupdate=False) cwd=oe_path, env=env, liveupdate=False)
os.remove(get_bb_env_file) os.remove(get_bb_env_file)
@ -163,65 +197,77 @@ def get_oe_environ(config, build_dir):
try: try:
(key, val) = line.split('=', 1) (key, val) = line.split('=', 1)
env[key] = val env[key] = val
except: except ValueError:
pass pass
vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR'] env_vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR']
if 'BB_ENV_EXTRAWHITE' in env: if 'BB_ENV_EXTRAWHITE' in env:
ew = env['BB_ENV_EXTRAWHITE'] + ' '.join(vars) extra_white = env['BB_ENV_EXTRAWHITE'] + ' '.join(env_vars)
env.update({'BB_ENV_EXTRAWHITE': ew}) env.update({'BB_ENV_EXTRAWHITE': extra_white})
vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK', env_vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK',
'SHELL', 'TERM']) 'SHELL', 'TERM'])
for v in vars: for env_var in env_vars:
if v in os.environ: if env_var in os.environ:
env[v] = os.environ[v] env[env_var] = os.environ[env_var]
return env return env
def ssh_add_key(env, key): def ssh_add_key(env, key):
p = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None, """
stderr=PIPE, env=env) Add ssh key to the ssh-agent
error = p.communicate(input=str.encode(key))[1] """
if p.returncode and error: process = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None,
logging.error('failed to add ssh key: {}'.format(error)) stderr=PIPE, env=env)
(_, error) = process.communicate(input=str.encode(key))
if process.returncode and error:
logging.error('failed to add ssh key: %s', error)
def ssh_cleanup_agent(config): def ssh_cleanup_agent(config):
"""Removes the identities and stop the ssh-agent instance """ """
Removes the identities and stop the ssh-agent instance
"""
# remove the identities # remove the identities
p = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ) process = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ)
p.wait() process.wait()
if p.returncode != 0: if process.returncode != 0:
logging.error('failed to delete SSH identities') logging.error('failed to delete SSH identities')
# stop the ssh-agent # stop the ssh-agent
p = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ) process = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ)
p.wait() process.wait()
if p.returncode != 0: if process.returncode != 0:
logging.error('failed to stop SSH agent') logging.error('failed to stop SSH agent')
def ssh_setup_agent(config, envkeys=['SSH_PRIVATE_KEY']): def ssh_setup_agent(config, envkeys=None):
"""
Starts the ssh-agent
"""
envkeys = envkeys or ['SSH_PRIVATE_KEY']
output = os.popen('/usr/bin/ssh-agent -s').readlines() output = os.popen('/usr/bin/ssh-agent -s').readlines()
for line in output: for line in output:
matches = re.search("(\S+)\=(\S+)\;", line) matches = re.search(r"(\S+)\=(\S+)\;", line)
if matches: if matches:
config.environ[matches.group(1)] = matches.group(2) config.environ[matches.group(1)] = matches.group(2)
for ek in envkeys: for envkey in envkeys:
key = os.environ.get(ek) key = os.environ.get(envkey)
if key: if key:
ssh_add_key(config.environ, key) ssh_add_key(config.environ, key)
else: else:
logging.warning('{} is missing'.format(ek)) logging.warning('%s is missing', envkey)
def ssh_no_host_key_check(config): def ssh_no_host_key_check(_):
"""
Disables ssh host key check
"""
home = os.path.expanduser('~') home = os.path.expanduser('~')
if not os.path.exists(home + '/.ssh'): if not os.path.exists(home + '/.ssh'):
os.mkdir(home + '/.ssh') os.mkdir(home + '/.ssh')
with open(home + '/.ssh/config', 'w') as f: with open(home + '/.ssh/config', 'w') as fds:
f.write('Host *\n\tStrictHostKeyChecking no\n\n') fds.write('Host *\n\tStrictHostKeyChecking no\n\n')

View File

@ -19,6 +19,9 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module contains the Repo class.
"""
import os import os
from urllib.parse import urlparse from urllib.parse import urlparse
@ -28,6 +31,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Repo: class Repo:
"""
Represents a repository in the kas configuration.
"""
def __init__(self, url, path, refspec=None, sublayers=None): def __init__(self, url, path, refspec=None, sublayers=None):
self.url = url self.url = url
self.path = path self.path = path
@ -37,6 +44,9 @@ class Repo:
self.git_operation_disabled = False self.git_operation_disabled = False
def disable_git_operations(self): def disable_git_operations(self):
"""
Disabled all git operation for this repository.
"""
self.git_operation_disabled = True self.git_operation_disabled = True
def __getattr__(self, item): def __getattr__(self, item):
@ -47,11 +57,12 @@ class Repo:
return [self.path + '/' + l for l in self.sublayers] return [self.path + '/' + l for l in self.sublayers]
elif item == 'qualified_name': elif item == 'qualified_name':
url = urlparse(self.url) url = urlparse(self.url)
return ('{url.netloc}{url.path}'.format(url=url) return ('{url.netloc}{url.path}'
.replace('@', '.') .format(url=url)
.replace(':', '.') .replace('@', '.')
.replace('/', '.') .replace(':', '.')
.replace('*', '.')) .replace('/', '.')
.replace('*', '.'))
def __str__(self): def __str__(self):
return '%s:%s %s' % (self.url, self.refspec, self.sublayers) return '%s:%s %s' % (self.url, self.refspec, self.sublayers)

View File

@ -19,6 +19,10 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
This module contains a kas plugin that opens a shell within the kas
environment
"""
import subprocess import subprocess
from kas.config import load_config from kas.config import load_config
@ -29,6 +33,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017'
class Shell: class Shell:
"""
Implements a kas plugin that opens a shell within the kas environment.
"""
def __init__(self, parser): def __init__(self, parser):
sh_prs = parser.add_parser('shell') sh_prs = parser.add_parser('shell')
@ -45,6 +53,11 @@ class Shell:
default='') default='')
def run(self, args): def run(self, args):
"""
Runs this kas plugin
"""
# pylint: disable= no-self-use
if args.cmd != 'shell': if args.cmd != 'shell':
return False return False
@ -63,8 +76,12 @@ class Shell:
class ShellCommand(Command): class ShellCommand(Command):
"""
This class implements the command that starts a shell.
"""
def __init__(self, cmd): def __init__(self, cmd):
Command.__init__(self) super().__init__()
self.cmd = [] self.cmd = []
if cmd: if cmd:
self.cmd = cmd self.cmd = cmd

View File

@ -19,18 +19,21 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
"""
Setup script for kas, a setup tool for bitbake based projects
"""
from setuptools import setup, find_packages
from os import path from os import path
from setuptools import setup, find_packages
from kas import __version__ from kas import __version__
__license__ = 'MIT' __license__ = 'MIT'
__copyright__ = 'Copyright (c) Siemens AG, 2017' __copyright__ = 'Copyright (c) Siemens AG, 2017'
here = path.abspath(path.dirname(__file__)) HERE = path.abspath(path.dirname(__file__))
with open(path.join(here, 'README.md')) as f: with open(path.join(HERE, 'README.md')) as f:
long_description = f.read() LONG_DESCRIPTION = f.read()
setup( setup(
@ -38,7 +41,7 @@ setup(
version=__version__, version=__version__,
description='Setup tool for bitbake based projects', description='Setup tool for bitbake based projects',
long_description=long_description, long_description=LONG_DESCRIPTION,
license='MIT', license='MIT',