diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..20c2ad2 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=too-few-public-methods,locally-disabled + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/README.md b/README.md index 0b1da11..358f6bc 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ Dependencies & installation This projects depends on - Python 3 -- PyYAML +- distro Python 3 package +- PyYAML Python 3 package If you need Python 2 support consider sending patches. The most obvious place to start is to use the trollius package intead of diff --git a/kas/__init__.py b/kas/__init__.py index bbea7d8..a6225fd 100644 --- a/kas/__init__.py +++ b/kas/__init__.py @@ -19,6 +19,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + kas - setup tool for bitbake based projects +""" from .__version__ import __version__ diff --git a/kas/__main__.py b/kas/__main__.py index 9f94acf..8a095ee 100644 --- a/kas/__main__.py +++ b/kas/__main__.py @@ -21,6 +21,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + The main entry point of kas, a setup tool for bitbake based projects +""" from .kas import main diff --git a/kas/__version__.py b/kas/__version__.py index e4e49b3..e90d963 100644 --- a/kas/__version__.py +++ b/kas/__version__.py @@ -1 +1,28 @@ +# kas - setup tool for bitbake based projects +# +# Copyright (c) Siemens AG, 2017 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +""" + This module contains the version of kas. +""" +__license__ = 'MIT' +__copyright__ = 'Copyright (c) Siemens AG, 2017' + __version__ = '0.9.0' diff --git a/kas/build.py b/kas/build.py index 196e46d..798288f 100644 --- a/kas/build.py +++ b/kas/build.py @@ -19,6 +19,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + The build plugin for kas. +""" import os from .config import load_config @@ -33,6 +36,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class Build: + """ + This class implements the build plugin for kas. + """ + def __init__(self, parser): bld_psr = parser.add_parser('build') @@ -49,6 +56,11 @@ class Build: default=[]) def run(self, args): + """ + Executes the build command of the kas plugin. + """ + # pylint: disable=no-self-use + if args.cmd != 'build': return False @@ -82,14 +94,21 @@ class Build: class BuildCommand(Command): + """ + Implement the bitbake build step. + """ + def __init__(self, task): - Command.__init__ + super().__init__() self.task = task def __str__(self): return 'build' def execute(self, config): + """ + Executes the bitbake build command. + """ # Start bitbake build of image bitbake = find_program(config.environ['PATH'], 'bitbake') run_cmd([bitbake, '-k', config.get_bitbake_target(), '-c', self.task], diff --git a/kas/config.py b/kas/config.py index 0125477..0e64b7e 100644 --- a/kas/config.py +++ b/kas/config.py @@ -19,14 +19,31 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module contains the implementation of the kas configuration. +""" import os import sys import logging import errno import json -import platform import yaml + +try: + from distro import id as get_distro_id +except ImportError: + import platform + + def get_distro_id(): + """ + Wrapper around platform.dist to simulate distro.id + platform.dist is deprecated and will be removed in python 3.7 + Use the 'distro' package instead. + """ + # pylint: disable=deprecated-method + return platform.dist()[0] + from .repos import Repo from .libkas import run_cmd @@ -35,24 +52,39 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class Config: + """ + This is an abstract class, that defines the interface of the + kas configuration. + """ def __init__(self): self.__kas_work_dir = os.environ.get('KAS_WORK_DIR', os.getcwd()) + self.environ = {} @property def build_dir(self): + """ + The path of the build directory. + """ return os.path.join(self.__kas_work_dir, 'build') @property def kas_work_dir(self): + """ + The path to the kas work directory. + """ return self.__kas_work_dir def setup_environ(self): - (distro, version, id) = platform.dist() - if distro in ['fedora', 'SuSE']: + """ + Sets the environment variables for process that are + started by kas. + """ + distro_id = get_distro_id() + if distro_id in ['fedora', 'SuSE']: self.environ = {'LC_ALL': 'en_US.utf8', 'LANG': 'en_US.utf8', 'LANGUAGE': 'en_US'} - elif distro in ['Ubuntu', 'debian']: + elif distro_id in ['Ubuntu', 'debian']: self.environ = {'LC_ALL': 'en_US.UTF-8', 'LANG': 'en_US.UTF-8', 'LANGUAGE': 'en_US:en'} @@ -61,17 +93,27 @@ class Config: self.environ = {} def get_repo_ref_dir(self): + """ + The path to the directory that contains the repository references. + """ + # pylint: disable=no-self-use + return os.environ.get('KAS_REPO_REF_DIR', None) class ConfigPython(Config): + """ + Implementation of a configuration that uses a Python script. + """ def __init__(self, filename, target): + # pylint: disable=exec-used + super().__init__() self.filename = os.path.abspath(filename) try: - with open(self.filename) as file: + with open(self.filename) as fds: env = {} - data = file.read() + data = fds.read() exec(data, env) self._config = env except IOError: @@ -82,80 +124,119 @@ class ConfigPython(Config): self.setup_environ() def __str__(self): - s = 'target: {}\n'.format(self.target) - s += 'repos:\n' - for r in self.get_repos(): - s += ' {}\n'.format(r.__str__()) - s += 'environ:\n' - for k, v in self.environ.items(): - s += ' {} = {}\n'.format(k, v) - s += 'proxy:\n' - for k, v in self.get_proxy_config().items(): - s += ' {} = {}\n'.format(k, v) - return s + output = 'target: {}\n'.format(self.target) + output += 'repos:\n' + for repo in self.get_repos(): + output += ' {}\n'.format(repo.__str__()) + output += 'environ:\n' + for key, value in self.environ.items(): + output += ' {} = {}\n'.format(key, value) + output += 'proxy:\n' + for key, value in self.get_proxy_config().items(): + output += ' {} = {}\n'.format(key, value) + return output def pre_hook(self, fname): + """ + Returns a function that is executed before every command or None. + """ try: self._config[fname + '_prepend'](self) except KeyError: pass def post_hook(self, fname): + """ + Returs a function that is executed after every command or None. + """ try: self._config[fname + '_append'](self) except KeyError: pass def get_hook(self, fname): + """ + Returns a function that is executed instead of the command or None. + """ try: return self._config[fname] except KeyError: return None def create_config(self, target): + """ + Sets the configuration for `target` + """ self.target = target self.repos = self._config['get_repos'](self, target) def get_proxy_config(self): + """ + Returns the proxy settings + """ return self._config['get_proxy_config']() def get_repos(self): + """ + Returns the list of repos + """ return iter(self.repos) def get_target(self): + """ + Returns the target + """ return self.target def get_bitbake_target(self): + """ + Return the bitbake target + """ try: return self._config['get_bitbake_target'](self) except KeyError: return self.target def get_bblayers_conf_header(self): + """ + Returns the bblayers.conf header + """ try: return self._config['get_bblayers_conf_header']() except KeyError: return '' def get_local_conf_header(self): + """ + Returns the local.conf header + """ try: return self._config['get_local_conf_header']() - except: + except KeyError: return '' def get_machine(self): + """ + Returns the machine + """ try: return self._config['get_machine'](self) except KeyError: return 'qemu' def get_distro(self): + """ + Returns the distro + """ try: return self._config['get_distro'](self) except KeyError: return 'poky' def get_gitlabci_config(self): + """ + Returns the GitlabCI configuration + """ try: return self._config['get_gitlabci_config'](self) except KeyError: @@ -163,21 +244,37 @@ class ConfigPython(Config): class ConfigStatic(Config): - def __init__(self, filename, target): + """ + An abstract class for static configuration files + """ + + def __init__(self, filename, _): super().__init__() self.filename = os.path.abspath(filename) - self._config = [] + self._config = {} - def pre_hook(self, target): + def pre_hook(self, _): + """ + Not used + """ pass - def post_hook(self, target): + def post_hook(self, _): + """ + Not used + """ pass - def get_hook(self, fname): - return None + def get_hook(self, _): + """ + Not used + """ + pass def get_proxy_config(self): + """ + Returns the proxy settings + """ try: return self._config['proxy_config'] except KeyError: @@ -186,6 +283,9 @@ class ConfigStatic(Config): 'no_proxy': os.environ.get('no_proxy', '')} def get_repos(self): + """ + Returns the list of repos + """ repos = [] for repo in self._config['repos']: try: @@ -196,57 +296,75 @@ class ConfigStatic(Config): url = repo['url'] if url == '': # in-tree configuration - (rc, output) = run_cmd(['/usr/bin/git', - 'rev-parse', - '--show-toplevel'], - cwd=os.path.dirname(self.filename), - env=self.environ) + (_, output) = run_cmd(['/usr/bin/git', + 'rev-parse', + '--show-toplevel'], + cwd=os.path.dirname(self.filename), + env=self.environ) url = output.strip() - r = Repo(url=url, - path=url, - sublayers=sublayers) - r.disable_git_operations() + rep = Repo(url=url, + path=url, + sublayers=sublayers) + rep.disable_git_operations() else: name = os.path.basename(os.path.splitext(url)[0]) - r = Repo(url=url, - path=os.path.join(self.kas_work_dir, name), - refspec=repo['refspec'], - sublayers=sublayers) - repos.append(r) + rep = Repo(url=url, + path=os.path.join(self.kas_work_dir, name), + refspec=repo['refspec'], + sublayers=sublayers) + repos.append(rep) return repos def get_bitbake_target(self): + """ + Return the bitbake target + """ try: return self._config['target'] except KeyError: return 'core-image-minimal' def get_bblayers_conf_header(self): + """ + Returns the bblayers.conf header + """ try: return self._config['bblayers_conf_header'] except KeyError: return '' def get_local_conf_header(self): + """ + Returns the local.conf header + """ try: return self._config['local_conf_header'] except KeyError: return '' def get_machine(self): + """ + Returns the machine + """ try: return self._config['machine'] except KeyError: return 'qemu' def get_distro(self): + """ + Returns the distro + """ try: return self._config['distro'] except KeyError: return 'poky' def get_gitlabci_config(self): + """ + Returns the GitlabCI configuration + """ try: return self._config['gitlabci_config'] except KeyError: @@ -254,47 +372,60 @@ class ConfigStatic(Config): class ConfigJson(ConfigStatic): + """ + Implements the configuration based on JSON files + """ + def __init__(self, filename, target): super().__init__(filename, target) self.filename = os.path.abspath(filename) try: - with open(self.filename, 'r') as f: - self._config = json.load(f) + with open(self.filename, 'r') as fds: + self._config = json.load(fds) except json.decoder.JSONDecodeError as msg: - logging.error('Could not load JSON config: {}'.format(msg)) + logging.error('Could not load JSON config: %s', msg) sys.exit(1) self.setup_environ() def get_bblayers_conf_header(self): - list = super().get_bblayers_conf_header() + header_list = super().get_bblayers_conf_header() conf = '' - for line in list: + for line in header_list: conf += str(line) + '\n' return conf def get_local_conf_header(self): - list = super().get_local_conf_header() + header_list = super().get_local_conf_header() conf = '' - for line in list: + for line in header_list: conf += str(line) + '\n' return conf class ConfigYaml(ConfigStatic): + """ + Implements the configuration based on Yaml files + """ + def __init__(self, filename, target): super().__init__(filename, target) self.filename = os.path.abspath(filename) try: - with open(self.filename, 'r') as f: - self._config = yaml.load(f) + with open(self.filename, 'r') as fds: + self._config = yaml.load(fds) except yaml.loader.ParserError as msg: - logging.error('Could not load YAML config: {}'.format(msg)) + logging.error('Could not load YAML config: %s', msg) sys.exit(1) self.setup_environ() def load_config(filename, target): - f, ext = os.path.splitext(filename) + """ + Return configuration generated from `filename`. + """ + # pylint: disable=redefined-variable-type + + (_, ext) = os.path.splitext(filename) if ext == '.py': cfg = ConfigPython(filename, target) elif ext == '.json': diff --git a/kas/kas.py b/kas/kas.py index a32b3bf..22389b5 100644 --- a/kas/kas.py +++ b/kas/kas.py @@ -21,6 +21,10 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module is the main entry point for kas, setup tool for bitbake based + projects +""" import argparse import traceback @@ -31,9 +35,9 @@ import pkg_resources try: import colorlog - have_colorlog = True + HAVE_COLORLOG = True except ImportError: - have_colorlog = False + HAVE_COLORLOG = False from .build import Build from .shell import Shell @@ -44,27 +48,34 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' def create_logger(): + """ + Setup the logging environment + """ log = logging.getLogger() # root logger log.setLevel(logging.INFO) - format = '%(asctime)s - %(levelname)-8s - %(message)s' + format_str = '%(asctime)s - %(levelname)-8s - %(message)s' date_format = '%Y-%m-%d %H:%M:%S' - if have_colorlog and os.isatty(2): - cformat = '%(log_color)s' + format + if HAVE_COLORLOG and os.isatty(2): + cformat = '%(log_color)s' + format_str colors = {'DEBUG': 'reset', 'INFO': 'reset', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red', 'CRITICAL': 'bold_red'} - f = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors) + formatter = colorlog.ColoredFormatter(cformat, date_format, + log_colors=colors) else: - f = logging.Formatter(format, date_format) - ch = logging.StreamHandler() - ch.setFormatter(f) - log.addHandler(ch) + formatter = logging.Formatter(format_str, date_format) + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + log.addHandler(stream_handler) return logging.getLogger(__name__) def kas(argv): + """ + The main entry point of kas. + """ create_logger() parser = argparse.ArgumentParser(description='Steer ebs-yocto builds') @@ -96,10 +107,15 @@ def kas(argv): def main(): + """ + The main function that operates as a wrapper around kas. + """ + # pylint: disable=broad-except + try: sys.exit(kas(sys.argv[1:])) except Exception as err: - logging.error('%s' % err) + logging.error('%s', err) traceback.print_exc() sys.exit(1) diff --git a/kas/libcmds.py b/kas/libcmds.py index 6dcef1f..70733c5 100644 --- a/kas/libcmds.py +++ b/kas/libcmds.py @@ -19,12 +19,14 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module contain common commands used by kas plugins. +""" import tempfile import logging import shutil import os -from urllib.parse import urlparse from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check, run_cmd, get_oe_environ) @@ -33,41 +35,63 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class Macro: + """ + Contains commands and provide method to run them. + """ def __init__(self): self.commands = [] def add(self, command): + """ + Appends commands to the command list. + """ self.commands.append(command) - def run(self, config, skip=[]): - for c in self.commands: - name = str(c) - if name in skip: + def run(self, config, skip=None): + """ + Runs command from the command list respective to the configuration. + """ + skip = skip or [] + for command in self.commands: + command_name = str(command) + if command_name in skip: continue - pre = config.pre_hook(name) - if pre: - logging.debug('execute ' + pre) - pre(config) - cmd = config.get_hook(name) - if cmd: - logging.debug('execute ' + cmd) - cmd(config) + pre_hook = config.pre_hook(command_name) + if pre_hook: + logging.debug('execute %s', pre_hook) + pre_hook(config) + command_hook = config.get_hook(command_name) + if command_hook: + logging.debug('execute %s', command_hook) + command_hook(config) else: - logging.debug('execute ' + str(c)) - c.execute(config) - post = config.post_hook(name) - if post: - logging.debug('execute ' + post) - post(config) + logging.debug('execute %s', command_name) + command.execute(config) + post_hook = config.post_hook(command_name) + if post_hook: + logging.debug('execute %s', post_hook) + post_hook(config) class Command: + """ + An abstract class that defines the interface of a command. + """ + def execute(self, config): + """ + This method executes the command. + """ pass class SetupHome(Command): + """ + Setups the home directory of kas. + """ + def __init__(self): + super().__init__() self.tmpdirname = tempfile.mkdtemp() def __del__(self): @@ -77,14 +101,18 @@ class SetupHome(Command): return 'setup_home' def execute(self, config): - with open(self.tmpdirname + '/.wgetrc', 'w') as f: - f.write('\n') - with open(self.tmpdirname + '/.netrc', 'w') as f: - f.write('\n') + with open(self.tmpdirname + '/.wgetrc', 'w') as fds: + fds.write('\n') + with open(self.tmpdirname + '/.netrc', 'w') as fds: + fds.write('\n') config.environ['HOME'] = self.tmpdirname class SetupDir(Command): + """ + Creates the build directory. + """ + def __str__(self): return 'setup_dir' @@ -95,6 +123,10 @@ class SetupDir(Command): class SetupSSHAgent(Command): + """ + Setup the ssh agent configuration. + """ + def __str__(self): return 'setup_ssh_agent' @@ -104,7 +136,9 @@ class SetupSSHAgent(Command): class CleanupSSHAgent(Command): - """Remove all the identities and stop the ssh-agent instance""" + """ + Remove all the identities and stop the ssh-agent instance. + """ def __str__(self): return 'cleanup_ssh_agent' @@ -114,6 +148,10 @@ class CleanupSSHAgent(Command): class SetupProxy(Command): + """ + Setups proxy configuration in the kas environment. + """ + def __str__(self): return 'setup_proxy' @@ -122,6 +160,10 @@ class SetupProxy(Command): class SetupEnviron(Command): + """ + Setups the kas environment. + """ + def __str__(self): return 'setup_environ' @@ -130,34 +172,39 @@ class SetupEnviron(Command): class WriteConfig(Command): + """ + Writes bitbake configuration files into the build directory. + """ + def __str__(self): return 'write_config' def execute(self, config): - self._write_bblayers_conf(config) - self._write_local_conf(config) + def _write_bblayers_conf(config): + filename = config.build_dir + '/conf/bblayers.conf' + with open(filename, 'w') as fds: + fds.write(config.get_bblayers_conf_header()) + fds.write('BBLAYERS ?= " \\\n') + for repo in config.get_repos(): + fds.write(' \\\n'.join(repo.layers + [''])) + fds.write('"\n') - def _append_layers(self, config, file): - for repo in config.get_repos(): - file.write(' \\\n'.join(repo.layers + [''])) + def _write_local_conf(config): + filename = config.build_dir + '/conf/local.conf' + with open(filename, 'w') as fds: + fds.write(config.get_local_conf_header()) + fds.write('MACHINE ?= "{}"\n'.format(config.get_machine())) + fds.write('DISTRO ?= "{}"\n'.format(config.get_distro())) - def _write_bblayers_conf(self, config): - filename = config.build_dir + '/conf/bblayers.conf' - with open(filename, 'w') as file: - file.write(config.get_bblayers_conf_header()) - file.write('BBLAYERS ?= " \\\n') - self._append_layers(config, file) - file.write('"\n') - - def _write_local_conf(self, config): - filename = config.build_dir + '/conf/local.conf' - with open(filename, 'w') as file: - file.write(config.get_local_conf_header()) - file.write('MACHINE ?= "{}"\n'.format(config.get_machine())) - file.write('DISTRO ?= "{}"\n'.format(config.get_distro())) + _write_bblayers_conf(config) + _write_local_conf(config) class ReposFetch(Command): + """ + Fetches repositories defined in the configuration + """ + def __str__(self): return 'repos_fetch' @@ -170,8 +217,7 @@ class ReposFetch(Command): os.makedirs(os.path.dirname(repo.path), exist_ok=True) gitsrcdir = os.path.join(config.get_repo_ref_dir() or '', repo.qualified_name) - logging.debug('Looking for repo ref dir in {}'. - format(gitsrcdir)) + logging.debug('Looking for repo ref dir in %s', gitsrcdir) if config.get_repo_ref_dir() and os.path.exists(gitsrcdir): run_cmd(['/usr/bin/git', 'clone', @@ -187,22 +233,26 @@ class ReposFetch(Command): continue # Does refspec in the current repository? - (rc, output) = run_cmd(['/usr/bin/git', 'cat-file', - '-t', repo.refspec], env=config.environ, - cwd=repo.path, fail=False) - if rc == 0: + (retc, output) = run_cmd(['/usr/bin/git', 'cat-file', + '-t', repo.refspec], env=config.environ, + cwd=repo.path, fail=False) + if retc == 0: continue # No it is missing, try to fetch - (rc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'], - env=config.environ, - cwd=repo.path, fail=False) - if rc: - logging.warning('Could not update repository {}: {}'. - format(repo.name, output)) + (retc, output) = run_cmd(['/usr/bin/git', 'fetch', '--all'], + env=config.environ, + cwd=repo.path, fail=False) + if retc: + logging.warning('Could not update repository %s: %s', + repo.name, output) class ReposCheckout(Command): + """ + Ensures that the right revision of each repo is check out. + """ + def __str__(self): return 'repos_checkout' @@ -212,22 +262,21 @@ class ReposCheckout(Command): continue # Check if repos is dirty - (rc, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'], - env=config.environ, cwd=repo.path, - fail=False) + (_, output) = run_cmd(['/usr/bin/git', 'diff', '--shortstat'], + env=config.environ, cwd=repo.path, + fail=False) if len(output): - logging.warning('Repo {} is dirty. no checkout'. - format(repo.name)) + logging.warning('Repo %s is dirty. no checkout', repo.name) continue # Check if current HEAD is what in the config file is defined. - (rc, output) = run_cmd(['/usr/bin/git', 'rev-parse', - '--verify', 'HEAD'], env=config.environ, - cwd=repo.path) + (_, output) = run_cmd(['/usr/bin/git', 'rev-parse', + '--verify', 'HEAD'], + env=config.environ, cwd=repo.path) if output.strip() == repo.refspec: - logging.info(('Repo {} has already checkout out correct ' - 'refspec. nothing to do').format(repo.name)) + logging.info('Repo %s has already checkout out correct ' + 'refspec. nothing to do', repo.name) continue run_cmd(['/usr/bin/git', 'checkout', '-q', diff --git a/kas/libkas.py b/kas/libkas.py index b2e6dd0..53fc8e0 100644 --- a/kas/libkas.py +++ b/kas/libkas.py @@ -19,6 +19,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module contains the core implementation of kas. +""" import re import os @@ -33,38 +36,58 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class LogOutput: + """ + Handles the log output of executed applications + """ def __init__(self, live): self.live = live self.stdout = [] self.stderr = [] def log_stdout(self, line): + """ + This method is called when a line over stdout is received. + """ if self.live: logging.info(line.strip()) self.stdout.append(line) def log_stderr(self, line): + """ + This method is called when a line over stderr is received. + """ if self.live: logging.error(line.strip()) self.stderr.append(line) @asyncio.coroutine -def _read_stream(stream, cb): +def _read_stream(stream, callback): + """ + This asynchronious method reads from the output stream of the + application and transfers each line to the callback function. + """ while True: line = yield from stream.readline() try: line = line.decode('utf-8') - except: - logging.warning('Could not decode line from stream - ignore it') + except UnicodeDecodeError as err: + logging.warning('Could not decode line from stream, ignore it: %s', + err) if line: - cb(line) + callback(line) else: break @asyncio.coroutine def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb): + """ + This function starts the subprocess, sets up the output stream + handlers and waits until the process has existed + """ + # pylint: disable=too-many-arguments + if shell: process = yield from asyncio.create_subprocess_shell( cmd, @@ -89,14 +112,18 @@ def _stream_subprocess(cmd, cwd, env, shell, stdout_cb, stderr_cb): return ret -def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True): - rc = 0 - stdout = [] - stderr = [] +def run_cmd(cmd, cwd, env=None, fail=True, shell=False, liveupdate=True): + """ + Starts a command. + """ + # pylint: disable=too-many-arguments + + env = env or {} + retc = 0 cmdstr = cmd if not shell: cmdstr = ' '.join(cmd) - logging.info('{}$ {}'.format(cwd, cmdstr)) + logging.info('%s$ %s', cwd, cmdstr) logo = LogOutput(liveupdate) if asyncio.get_event_loop().is_closed(): @@ -105,22 +132,25 @@ def run_cmd(cmd, cwd, env={}, fail=True, shell=False, liveupdate=True): else: loop = asyncio.get_event_loop() - rc = loop.run_until_complete( + retc = loop.run_until_complete( _stream_subprocess(cmd, cwd, env, shell, logo.log_stdout, logo.log_stderr)) loop.close() - if rc and fail: + if retc and fail: msg = 'Command "{cwd}$ {cmd}" failed\n'.format(cwd=cwd, cmd=cmdstr) for line in logo.stderr: msg += line logging.error(msg) - sys.exit(rc) + sys.exit(retc) - return (rc, ''.join(logo.stdout)) + return (retc, ''.join(logo.stdout)) def find_program(paths, name): + """ + Find a file within the paths array and returns its path. + """ for path in paths.split(os.pathsep): prg = os.path.join(path, name) if os.path.isfile(prg): @@ -129,6 +159,10 @@ def find_program(paths, name): def get_oe_environ(config, build_dir): + """ + Create the openembedded environment variables. + """ + # pylint: disable=too-many-locals # nasty side effect function: running oe-init-build-env also # creates the conf directory @@ -142,19 +176,19 @@ def get_oe_environ(config, build_dir): sys.exit(1) get_bb_env_file = tempfile.mktemp() - with open(get_bb_env_file, 'w') as f: + with open(get_bb_env_file, 'w') as fds: script = """#!/bin/bash source oe-init-build-env $1 > /dev/null 2>&1 env """ - f.write(script) + fds.write(script) os.chmod(get_bb_env_file, 0o775) env = {} env['PATH'] = '/bin:/usr/bin' - (rc, output) = run_cmd([get_bb_env_file, build_dir], - cwd=oe_path, env=env, liveupdate=False) + (_, output) = run_cmd([get_bb_env_file, build_dir], + cwd=oe_path, env=env, liveupdate=False) os.remove(get_bb_env_file) @@ -163,65 +197,77 @@ def get_oe_environ(config, build_dir): try: (key, val) = line.split('=', 1) env[key] = val - except: + except ValueError: pass - vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR'] + env_vars = ['SSTATE_DIR', 'DL_DIR', 'TMPDIR'] if 'BB_ENV_EXTRAWHITE' in env: - ew = env['BB_ENV_EXTRAWHITE'] + ' '.join(vars) - env.update({'BB_ENV_EXTRAWHITE': ew}) + extra_white = env['BB_ENV_EXTRAWHITE'] + ' '.join(env_vars) + env.update({'BB_ENV_EXTRAWHITE': extra_white}) - vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK', - 'SHELL', 'TERM']) + env_vars.extend(['SSH_AGENT_PID', 'SSH_AUTH_SOCK', + 'SHELL', 'TERM']) - for v in vars: - if v in os.environ: - env[v] = os.environ[v] + for env_var in env_vars: + if env_var in os.environ: + env[env_var] = os.environ[env_var] return env def ssh_add_key(env, key): - p = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None, - stderr=PIPE, env=env) - error = p.communicate(input=str.encode(key))[1] - if p.returncode and error: - logging.error('failed to add ssh key: {}'.format(error)) + """ + Add ssh key to the ssh-agent + """ + process = Popen(['/usr/bin/ssh-add', '-'], stdin=PIPE, stdout=None, + stderr=PIPE, env=env) + (_, error) = process.communicate(input=str.encode(key)) + if process.returncode and error: + logging.error('failed to add ssh key: %s', error) def ssh_cleanup_agent(config): - """Removes the identities and stop the ssh-agent instance """ + """ + Removes the identities and stop the ssh-agent instance + """ # remove the identities - p = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ) - p.wait() - if p.returncode != 0: + process = Popen(['/usr/bin/ssh-add', '-D'], env=config.environ) + process.wait() + if process.returncode != 0: logging.error('failed to delete SSH identities') # stop the ssh-agent - p = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ) - p.wait() - if p.returncode != 0: + process = Popen(['/usr/bin/ssh-agent', '-k'], env=config.environ) + process.wait() + if process.returncode != 0: logging.error('failed to stop SSH agent') -def ssh_setup_agent(config, envkeys=['SSH_PRIVATE_KEY']): +def ssh_setup_agent(config, envkeys=None): + """ + Starts the ssh-agent + """ + envkeys = envkeys or ['SSH_PRIVATE_KEY'] output = os.popen('/usr/bin/ssh-agent -s').readlines() for line in output: - matches = re.search("(\S+)\=(\S+)\;", line) + matches = re.search(r"(\S+)\=(\S+)\;", line) if matches: config.environ[matches.group(1)] = matches.group(2) - for ek in envkeys: - key = os.environ.get(ek) + for envkey in envkeys: + key = os.environ.get(envkey) if key: ssh_add_key(config.environ, key) else: - logging.warning('{} is missing'.format(ek)) + logging.warning('%s is missing', envkey) -def ssh_no_host_key_check(config): +def ssh_no_host_key_check(_): + """ + Disables ssh host key check + """ home = os.path.expanduser('~') if not os.path.exists(home + '/.ssh'): os.mkdir(home + '/.ssh') - with open(home + '/.ssh/config', 'w') as f: - f.write('Host *\n\tStrictHostKeyChecking no\n\n') + with open(home + '/.ssh/config', 'w') as fds: + fds.write('Host *\n\tStrictHostKeyChecking no\n\n') diff --git a/kas/repos.py b/kas/repos.py index 0244751..5af1038 100644 --- a/kas/repos.py +++ b/kas/repos.py @@ -19,6 +19,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module contains the Repo class. +""" import os from urllib.parse import urlparse @@ -28,6 +31,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class Repo: + """ + Represents a repository in the kas configuration. + """ + def __init__(self, url, path, refspec=None, sublayers=None): self.url = url self.path = path @@ -37,6 +44,9 @@ class Repo: self.git_operation_disabled = False def disable_git_operations(self): + """ + Disabled all git operation for this repository. + """ self.git_operation_disabled = True def __getattr__(self, item): @@ -47,11 +57,12 @@ class Repo: return [self.path + '/' + l for l in self.sublayers] elif item == 'qualified_name': url = urlparse(self.url) - return ('{url.netloc}{url.path}'.format(url=url) - .replace('@', '.') - .replace(':', '.') - .replace('/', '.') - .replace('*', '.')) + return ('{url.netloc}{url.path}' + .format(url=url) + .replace('@', '.') + .replace(':', '.') + .replace('/', '.') + .replace('*', '.')) def __str__(self): return '%s:%s %s' % (self.url, self.refspec, self.sublayers) diff --git a/kas/shell.py b/kas/shell.py index 688ab91..dd00beb 100644 --- a/kas/shell.py +++ b/kas/shell.py @@ -19,6 +19,10 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + This module contains a kas plugin that opens a shell within the kas + environment +""" import subprocess from kas.config import load_config @@ -29,6 +33,10 @@ __copyright__ = 'Copyright (c) Siemens AG, 2017' class Shell: + """ + Implements a kas plugin that opens a shell within the kas environment. + """ + def __init__(self, parser): sh_prs = parser.add_parser('shell') @@ -45,6 +53,11 @@ class Shell: default='') def run(self, args): + """ + Runs this kas plugin + """ + # pylint: disable= no-self-use + if args.cmd != 'shell': return False @@ -63,8 +76,12 @@ class Shell: class ShellCommand(Command): + """ + This class implements the command that starts a shell. + """ + def __init__(self, cmd): - Command.__init__(self) + super().__init__() self.cmd = [] if cmd: self.cmd = cmd diff --git a/setup.py b/setup.py index f619deb..f3a8de5 100644 --- a/setup.py +++ b/setup.py @@ -19,18 +19,21 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" + Setup script for kas, a setup tool for bitbake based projects +""" -from setuptools import setup, find_packages from os import path +from setuptools import setup, find_packages from kas import __version__ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017' -here = path.abspath(path.dirname(__file__)) -with open(path.join(here, 'README.md')) as f: - long_description = f.read() +HERE = path.abspath(path.dirname(__file__)) +with open(path.join(HERE, 'README.md')) as f: + LONG_DESCRIPTION = f.read() setup( @@ -38,7 +41,7 @@ setup( version=__version__, description='Setup tool for bitbake based projects', - long_description=long_description, + long_description=LONG_DESCRIPTION, license='MIT',