Commit 4313d121 authored by Daniele Venzano's avatar Daniele Venzano

Big push to fix all reasonable pylint messages

parent 3d21e03d
inherits:
- strictness_veryhigh
ignore:
- (^|/)\..+
- ^docs/
- ^build/
uses:
- flask
member-warning: true
output-format: grouped
dodgy:
# Looks at Python code to search for things which look "dodgy"
# such as passwords or git conflict artifacts
run: true
mccabe:
# complexity checking.
run: true
pep257:
# docstring checking
run: true
pep8:
# style checking
run: false
pyflakes:
# preferring 'frosted' instead (a fork of)
run: true
pylint:
# static analysis and then some, see .pylintrc file for configuration
run: true
pyroma:
# checks setup.py
run: false
vulture:
# this tool does a good job of finding unused code.
run: true
......@@ -15,7 +15,7 @@ persistent=yes
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=2
jobs=4
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
......@@ -47,7 +47,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=line-too-long
disable=line-too-long,logging-format-interpolation,too-few-public-methods,too-many-instance-attributes,fixme,too-many-branches,file-ignored,global-statement,redefined-variable-type,no-self-use
[REPORTS]
......@@ -68,7 +68,7 @@ init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=_$|dummy
dummy-variables-rgx=.*_$|dummy
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
......@@ -117,7 +117,7 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
ignored-modules=zmq,ldap
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
......@@ -158,7 +158,7 @@ ignore-imports=no
bad-functions=
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,e
good-names=i,j,k,e,ex,th,q,ip
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
......@@ -183,10 +183,10 @@ method-rgx=[a-z_][a-z0-9_]{2,30}$
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
module-rgx=(([a-z_][a-z0-9_\-]*)|([A-Z][a-zA-Z0-9_\-]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
module-name-hint=(([a-z_][a-z0-9_\-]*)|([A-Z][a-zA-Z0-9_\-]+))$
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
......@@ -225,10 +225,10 @@ const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|log)$
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__)|log)$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
attr-rgx=([a-z_][a-z0-9_]{2,30}|id)$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
attr-name-hint=([a-z_][a-z0-9_]{2,30}|id)$
# Regular expression which should only match function or class names that do
# not require a docstring.
......
......@@ -3,9 +3,8 @@ python:
- "3.4"
- "3.5"
install:
- pip install -r requirements.txt
- pip install pylint
- pip install -r requirements_tests.txt
before_script:
script:
- prospector
- pylint *.py zoe_*
- doc8 docs/
prospector[with_vulture]
pylint
vulture
doc8
sphinx
mypy-lang
......@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Bypass the Zoe scheduler to run a ZApp and leave the logs inside Docker, used for debugging ZApps."""
import datetime
import json
import logging
......@@ -30,23 +32,25 @@ from zoe_master.zapp_to_docker import execution_to_containers, terminate_executi
log = logging.getLogger("main")
LOG_FORMAT = '%(asctime)-15s %(levelname)s %(name)s (%(threadName)s): %(message)s'
config_paths = [
CONFIG_PATHS = [
'zoe.conf',
'/etc/zoe/zoe.conf'
]
class FakeSQLManager:
"""A fake in-memory state class."""
def __init__(self):
self.executions = []
self.services = []
self._last_id = 0
def execution_list(self, only_one=False, **kwargs):
"""Execution list."""
ret_list = []
for e in self.executions:
for k, v in kwargs.items():
if getattr(e, k) == v:
for key, value in kwargs.items():
if getattr(e, key) == value:
ret_list.append(e)
if only_one:
return ret_list[0]
......@@ -54,16 +58,18 @@ class FakeSQLManager:
return ret_list
def execution_update(self, exec_id, **kwargs):
"""Execution update."""
for e in self.executions:
if e.id == exec_id:
for k, v in kwargs.items():
if k == "status":
for key, value in kwargs.items():
if key == "status":
continue
print(k, v)
setattr(e, k, v)
print(key, value)
setattr(e, key, value)
break
def execution_new(self, name, user_id, description):
"""New execution."""
e_dict = {
'id': self._last_id,
'name': name,
......@@ -81,13 +87,15 @@ class FakeSQLManager:
return self._last_id - 1
def execution_delete(self, execution_id):
"""Delete execution."""
raise NotImplementedError
def service_list(self, only_one=False, **kwargs):
"""Service list."""
ret_list = []
for e in self.services:
for k, v in kwargs.items():
if getattr(e, k) == v:
for key, value in kwargs.items():
if getattr(e, key) == value:
ret_list.append(e)
if only_one:
return ret_list[0]
......@@ -95,13 +103,15 @@ class FakeSQLManager:
return ret_list
def service_update(self, service_id, **kwargs):
"""Service update."""
for e in self.services:
if e.id == service_id:
for k, v in kwargs.items():
setattr(e, k, v)
for key, value in kwargs.items():
setattr(e, key, value)
break
def service_new(self, execution_id, name, service_group, description):
"""Service new."""
s_dict = {
'id': self._last_id,
'name': name,
......@@ -112,16 +122,17 @@ class FakeSQLManager:
'service_group': service_group,
'error_message': None
}
s = Service(s_dict, self)
self.services.append(s)
service = Service(s_dict, self)
self.services.append(service)
self._last_id += 1
return self._last_id - 1
def load_configuration(test_conf=None):
"""Load configuration from the command line."""
if test_conf is None:
argparser = ArgumentParser(description="Zoe application tester - Container Analytics as a Service core component",
default_config_files=config_paths,
default_config_files=CONFIG_PATHS,
auto_env_var_prefix="ZOE_MASTER_",
args_for_setting_config_path=["--config"],
args_for_writing_out_config_file=["--write-config"])
......@@ -156,6 +167,7 @@ def load_configuration(test_conf=None):
def main():
"""The main entrypoint function."""
conf = load_configuration()
config.load_configuration(conf)
args = config.get_conf()
......@@ -171,7 +183,6 @@ def main():
logging.getLogger("tornado").setLevel(logging.DEBUG)
state = FakeSQLManager()
config.singletons['sql_manager'] = state
zapp_description = json.load(args.jsonfile)
......@@ -180,7 +191,7 @@ def main():
exec_id = state.execution_new('test', 'fake_user', zapp_description)
e = state.execution_list(only_one=True, id=exec_id)
_digest_application_description(e)
_digest_application_description(state, e)
print('Zapp digested, starting containers...')
execution_to_containers(e)
......
......@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""API component entry point."""
from zoe_api.entrypoint import zoe_web_main
if __name__ == "__main__":
......
......@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Master component entry point."""
from zoe_master.entrypoint import main
if __name__ == '__main__':
......
......@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command line client entry point."""
from zoe_cmd.entrypoint import zoe
if __name__ == "__main__":
......
......@@ -13,14 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""The real API, exposed as web pages or REST API."""
import logging
import re
from zoe_lib.config import get_conf
import zoe_api.master_api
import zoe_lib.sql_manager
import zoe_lib.applications
import zoe_lib.exceptions
import zoe_api.master_api
import zoe_api.exceptions
log = logging.getLogger(__name__)
......@@ -28,6 +31,8 @@ log = logging.getLogger(__name__)
class APIEndpoint:
"""
The APIEndpoint class.
:type master: zoe_api.master_api.APIManager
:type sql: zoe_lib.sql_manager.SQLManager
"""
......@@ -36,6 +41,7 @@ class APIEndpoint:
self.sql = zoe_lib.sql_manager.SQLManager(get_conf())
def execution_by_id(self, uid, role, execution_id) -> zoe_lib.sql_manager.Execution:
"""Lookup an execution by its ID."""
e = self.sql.execution_list(id=execution_id, only_one=True)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
......@@ -44,11 +50,13 @@ class APIEndpoint:
return e
def execution_list(self, uid, role, **filters):
"""Generate a optionally filtered list of executions."""
execs = self.sql.execution_list(**filters)
ret = [e for e in execs if e.user_id == uid or role == 'admin']
return ret
def execution_start(self, uid, role, exec_name, application_description):
def execution_start(self, uid, role_, exec_name, application_description):
"""Start an execution."""
try:
zoe_lib.applications.app_validate(application_description)
except zoe_lib.exceptions.InvalidApplicationDescription as e:
......@@ -62,10 +70,11 @@ class APIEndpoint:
new_id = self.sql.execution_new(exec_name, uid, application_description)
success, message = self.master.execution_start(new_id)
if not success:
raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up.')
raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up ({}).'.format(message))
return new_id
def execution_terminate(self, uid, role, exec_id):
"""Terminate an execution."""
e = self.sql.execution_list(id=exec_id, only_one=True)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
......@@ -79,6 +88,7 @@ class APIEndpoint:
raise zoe_api.exceptions.ZoeException('Execution is not running')
def execution_delete(self, uid, role, exec_id):
"""Delete an execution."""
e = self.sql.execution_list(id=exec_id, only_one=True)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
......@@ -97,19 +107,21 @@ class APIEndpoint:
raise zoe_api.exceptions.ZoeException(message)
def service_by_id(self, uid, role, service_id) -> zoe_lib.sql_manager.Service:
s = self.sql.service_list(id=service_id, only_one=True)
if s is None:
"""Lookup a service by its ID."""
service = self.sql.service_list(id=service_id, only_one=True)
if service is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
s_exec = self.sql.execution_list(only_one=True, id=s.execution_id)
s_exec = self.sql.execution_list(only_one=True, id=service.execution_id)
if s_exec.user_id != uid and role != 'admin':
raise zoe_api.exceptions.ZoeAuthException()
return s
return service
def retry_submit_error_executions(self):
"""Resubmit any execution forgotten by the master."""
waiting_execs = self.sql.execution_list(status=zoe_lib.sql_manager.Execution.SUBMIT_STATUS)
if waiting_execs is None or len(waiting_execs) == 0:
return
e = waiting_execs.pop(0)
e = waiting_execs[0]
success, message = self.master.execution_start(e.id)
if not success:
log.warning('Zoe Master unavailable, execution {} still waiting'.format(e.id))
log.warning('Zoe Master unavailable ({}), execution {} still waiting'.format(message, e.id))
......@@ -13,7 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base authenticator class."""
class BaseAuthenticator:
"""Base authenticator class."""
def auth(self, username, password):
"""The methods that needs to be overridden by implementations."""
raise NotImplementedError
......@@ -12,6 +12,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""LDAP authentication module."""
import logging
import ldap
......@@ -23,6 +26,7 @@ log = logging.getLogger(__name__)
class LDAPAuthenticator(zoe_api.auth.base.BaseAuthenticator):
"""A simple LDAP authenticator."""
def __init__(self):
self.connection = ldap.initialize(get_conf().ldap_server_uri)
self.base_dn = get_conf().ldap_base_dn
......@@ -30,6 +34,7 @@ class LDAPAuthenticator(zoe_api.auth.base.BaseAuthenticator):
self.bind_password = get_conf().ldap_bind_password
def auth(self, username, password):
"""Authenticate the user or raise an exception."""
search_filter = "uid=" + username
uid = None
role = 'guest'
......@@ -46,7 +51,7 @@ class LDAPAuthenticator(zoe_api.auth.base.BaseAuthenticator):
elif get_conf().ldap_guest_gid in gid_numbers:
role = 'guest'
else:
log.warn('User {} has an unknown group ID ({}), using guest role'.format(username, result[0][1]['gidNumber']))
log.warning('User {} has an unknown group ID ({}), using guest role'.format(username, result[0][1]['gidNumber']))
role = 'guest'
except ldap.LDAPError:
log.exception("LDAP exception")
......
......@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database initialization."""
import psycopg2
import psycopg2.extras
......@@ -23,16 +25,19 @@ SQL_SCHEMA_VERSION = 0 # ---> Increment this value every time the schema change
def version_table(cur):
"""Create the version table."""
cur.execute("CREATE TABLE IF NOT EXISTS public.versions (deployment text, version integer)")
def schema(cur, deployment_name):
"""Create the schema for the configured deployment name."""
cur.execute("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = %s)", (deployment_name,))
if not cur.fetchone()[0]:
cur.execute('CREATE SCHEMA %s', (deployment_name,))
def check_schema_version(cur, deployment_name):
"""Check if the schema version matches this source code version."""
cur.execute("SELECT version FROM public.versions WHERE deployment = %s", (deployment_name,))
row = cur.fetchone()
if row is None:
......@@ -47,6 +52,7 @@ def check_schema_version(cur, deployment_name):
def create_tables(cur):
"""Create the Zoe database tables."""
cur.execute('''CREATE TABLE execution (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
......@@ -72,6 +78,7 @@ def create_tables(cur):
def init():
"""DB init entrypoint."""
dsn = 'dbname=' + get_conf().dbname + \
' user=' + get_conf().dbuser + \
' password=' + get_conf().dbpass + \
......
......@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Zoe API entrypoint module."""
import logging
from flask import Flask
......
......@@ -13,12 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Exceptions used by the API component."""
class ZoeException(Exception):
"""
A generic exception.
"""
def __init__(self, message='Something happened'):
super().__init__()
self.message = message
def __str__(self):
......@@ -26,10 +29,12 @@ class ZoeException(Exception):
class ZoeAuthException(ZoeException):
"""An authentication error."""
pass
class ZoeNotFoundException(ZoeException):
"""Th euser is looking for an object that does not exist."""
pass
......
......@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""The client side of the ZeroMQ API."""
import logging
import zmq
......@@ -23,6 +25,7 @@ log = logging.getLogger(__name__)
class APIManager:
"""Main class for the API."""
REQUEST_TIMEOUT = 2500
REQUEST_RETRIES = 1
......@@ -72,6 +75,7 @@ class APIManager:
self._connect()
def execution_start(self, exec_id):
"""Start an execution."""
msg = {
'command': 'execution_start',
'exec_id': exec_id
......@@ -79,6 +83,7 @@ class APIManager:
return self._request_reply(msg)
def execution_terminate(self, exec_id):
"""Terminate an execution."""
msg = {
'command': 'execution_terminate',
'exec_id': exec_id
......@@ -86,15 +91,9 @@ class APIManager:
return self._request_reply(msg)
def execution_delete(self, exec_id):
"""Delete an execution."""
msg = {
'command': 'execution_delete',
'exec_id': exec_id
}
return self._request_reply(msg)
def service_inspect(self, service_id):
msg = {
'command': 'service_inspect',
'service_id': service_id
}
return self._request_reply(msg)
......@@ -13,21 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""RESTful Flask API definition."""
import sys
import pkgutil
from flask import Blueprint
from flask_restful import Api
from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI, ExecutionDeleteAPI
from zoe_api.rest_api.info import InfoAPI
from zoe_api.rest_api.service import ServiceAPI
from zoe_lib.version import ZOE_API_VERSION
from zoe_api.rest_api.query import QueryAPI
API_PATH = '/api/' + ZOE_API_VERSION
def api_init(api_endpoint) -> Blueprint:
"""Initialize the API"""
api_bp = Blueprint('api', __name__)
api = Api(api_bp, catch_all_404s=True)
......@@ -36,19 +40,17 @@ def api_init(api_endpoint) -> Blueprint:
api.add_resource(ExecutionDeleteAPI, API_PATH + '/execution/delete/<int:execution_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
api.add_resource(ExecutionCollectionAPI, API_PATH + '/execution', resource_class_kwargs={'api_endpoint': api_endpoint})
api.add_resource(ServiceAPI, API_PATH + '/service/<int:service_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
api.add_resource(QueryAPI, API_PATH + '/query', resource_class_kwargs={'api_endpoint': api_endpoint})
return api_bp
# Work around a Python 3.4.0 bug that affects Flask
if sys.version_info == (3, 4, 0, 'final', 0):
import pkgutil
orig_get_loader = pkgutil.get_loader
ORIG_LOADER = pkgutil.get_loader
def get_loader(name):
"""Wrap the original loader to catch a buggy exception."""
try:
return orig_get_loader(name)
return ORIG_LOADER(name)
except AttributeError:
pass
......
......@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Execution API endpoints."""
from flask_restful import Resource, request
from werkzeug.exceptions import BadRequest
......@@ -22,11 +24,14 @@ import zoe_api.api_endpoint
class ExecutionAPI(Resource):
"""The Execution API endpoint."""
def __init__(self, api_endpoint: zoe_api.api_endpoint.APIEndpoint):
self.api_endpoint = api_endpoint
@catch_exceptions
def get(self, execution_id):
"""GET a single execution by its ID."""
uid, role = get_auth(request)
e = self.api_endpoint.execution_by_id(uid, role, execution_id)
......@@ -36,6 +41,8 @@ class ExecutionAPI(Resource):
@catch_exceptions
def delete(self, execution_id: int):
"""
Terminate an execution.
:param execution_id: the execution to be terminated
:return:
"""
......@@ -49,12 +56,16 @@ class ExecutionAPI(Resource):
class ExecutionDeleteAPI(Resource):
"""The ExecutionDelete API endpoints."""
def __init__(self, api_endpoint: zoe_api.api_endpoint.APIEndpoint):
self.api_endpoint = api_endpoint
@catch_exceptions
def delete(self, execution_id: int):
"""
Delete an execution.
:param execution_id: the execution to be deleted
:return:
"""
......@@ -68,6 +79,8 @@ class ExecutionDeleteAPI(Resource):
class ExecutionCollectionAPI(Resource):
"""The Execution Collection API endpoints."""
def __init__(self, api_endpoint: zoe_api.api_endpoint.APIEndpoint):
self.api_endpoint = api_endpoint
......
......@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Info API endpoint."""
from flask_restful import Resource
import zoe_api.api_endpoint
......@@ -22,11 +24,13 @@ from zoe_lib.version import ZOE_API_VERSION, ZOE_APPLICATION_FORMAT_VERSION, ZOE
class InfoAPI(Resource):
"""The Info API endpoint."""
def __init__(self, api_endpoint: zoe_api.api_endpoint.APIEndpoint):
self.api_endpoint = api_endpoint
@catch_exceptions
def get(self):
"""HTTP GET method."""
ret = {
'version': ZOE_VERSION,
'api_version': ZOE_API_VERSION,
......
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restful import Resource, request
from werkzeug.exceptions import BadRequest
from zoe_api.exceptions import ZoeRestAPIException
import zoe_api.api_endpoint
from zoe_api.rest_api.utils import catch_exceptions, get_auth
class QueryAPI(Resource):
def __init__(self, api_endpoint: zoe_api.api_endpoint.APIEndpoint):