Fix pylint errors and warnings

Bump testing to Python 3.5 and postgresql 9.5 (ubuntu xenial)

.gitlab-ci.yml

@@ -20,7 +20,7 @@ cache:
 
 pylint:
   stage: static-test
-  image: python:3.4
+  image: python:3.5
   script:
     - pip install -U pip setuptools
     - pip install -r requirements.txt
@@ -30,8 +30,8 @@ pylint:
 unittests:
   stage: static-test
   services:
-    - postgres:9.3
-  image: python:3.4
+    - postgres:9.5
+  image: python:3.5
   script:
     - pip install -U pip setuptools
     - pip install -r requirements.txt
@@ -40,7 +40,7 @@ unittests:
 
 docs-test:
   stage: static-test
-  image: python:3.4
+  image: python:3.5
   script:
     - pip install -U pip setuptools
     - pip install -r requirements.txt
@@ -61,7 +61,7 @@ images:
 api-test:
   stage: integration-test
   services:
-    - postgres:9.3
+    - postgres:9.5
   image: zoerepo/${ZOE_TEST_IMAGE}
   script:
     - echo 'admin,admin,admin' > zoepass.csv
@@ -71,7 +71,7 @@ api-test:
 
 mirror-github:
   stage: deploy
-  image: python:3.4
+  image: python:3.5
   before_script:
     - which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )
     - eval $(ssh-agent -s)

create_db_tables.py

@@ -18,10 +18,10 @@
 import sys
 import time
 
-import zoe_lib.config as config
+import zoe_lib.config
 import zoe_lib.state.sql_manager
 
-config.load_configuration()
+zoe_lib.config.load_configuration()
 
 print("Warning, this script will delete the database tables for the deployment '{}' before creating new ones".format(config.get_conf().deployment_name))
 print("If you are installing Zoe for the first time, you have nothing to worry about")
@@ -33,4 +33,4 @@ except KeyboardInterrupt:
     print("Aborted.")
     sys.exit(1)
 
-zoe_lib.state.sql_manager.SQLManager(config.get_conf()).init_db(force=True)
+zoe_lib.state.sql_manager.SQLManager(zoe_lib.config.get_conf()).init_db(force=True)

zoe_api/entrypoint.py

@@ -22,7 +22,7 @@ from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
 from tornado.web import Application
 
-import zoe_lib.config as config
+import zoe_lib.config
 import zoe_lib.state
 import zoe_api.api_endpoint
 import zoe_api.rest_api
@@ -40,8 +40,8 @@ def zoe_web_main(test_conf=None) -> int:
     This is the entry point for the Zoe Web script.
     :return: int
     """
-    config.load_configuration(test_conf)
-    args = config.get_conf()
+    zoe_lib.config.load_configuration(test_conf)
+    args = zoe_lib.config.get_conf()
 
     log_args = {
         'level': logging.DEBUG if args.debug else logging.INFO,
@@ -53,7 +53,7 @@ def zoe_web_main(test_conf=None) -> int:
     logging.getLogger("MARKDOWN").setLevel(logging.WARNING)
     logging.getLogger("tornado").setLevel(logging.WARNING)
 
-    sql_manager = zoe_lib.state.SQLManager(config.get_conf())
+    sql_manager = zoe_lib.state.SQLManager(zoe_lib.config.get_conf())
     sql_manager.init_db()
 
     master_api = zoe_api.master_api.APIManager()
@@ -62,7 +62,7 @@ def zoe_web_main(test_conf=None) -> int:
     app_settings = {
         'static_path': os.path.join(os.path.dirname(__file__), "web", "static"),
         'template_path': os.path.join(os.path.dirname(__file__), "web", "templates"),
-        'cookie_secret': config.get_conf().cookie_secret,
+        'cookie_secret': zoe_lib.config.get_conf().cookie_secret,
         'login_url': '/login',
         'debug': args.debug
     }

zoe_api/master_api.py

@@ -20,7 +20,7 @@ from typing import Dict, Any, Tuple
 
 import zmq
 
-import zoe_lib.config as config
+import zoe_lib.config
 
 log = logging.getLogger(__name__)
 
@@ -36,7 +36,7 @@ class APIManager:
         self.context = zmq.Context(1)
         self.zmq_s = None
         self.poll = zmq.Poller()
-        self.master_uri = config.get_conf().master_url  # type: str
+        self.master_uri = zoe_lib.config.get_conf().master_url  # type: str
         self._connect()
 
     def _connect(self):

zoe_api/rest_api/execution.py

@@ -130,7 +130,7 @@ class ExecutionCollectionAPI(ZoeAPIRequestHandler):
             self.set_status(e.status_code, e.message)
             return
 
-        self.write(dict([(e.id, e.serialize()) for e in execs]))
+        self.write({e.id: e.serialize() for e in execs})
 
     def post(self):
         """

zoe_api/rest_api/quota.py

@@ -102,7 +102,7 @@ class QuotaCollectionAPI(ZoeAPIRequestHandler):
             self.set_status(e.status_code, e.message)
             return
 
-        self.write(dict([(r.id, r.serialize()) for r in quota]))
+        self.write({r.id: r.serialize() for r in quota})
 
     def post(self):
         """HTTP POST method."""

zoe_api/rest_api/role.py

@@ -102,7 +102,7 @@ class RoleCollectionAPI(ZoeAPIRequestHandler):
             self.set_status(e.status_code, e.message)
             return
 
-        self.write(dict([(r.id, r.serialize()) for r in role]))
+        self.write({r.id: r.serialize() for r in role})
 
     def post(self):
         """HTTP POST method."""

zoe_api/rest_api/user.py

@@ -118,7 +118,7 @@ class UserCollectionAPI(ZoeAPIRequestHandler):
             self.set_status(e.status_code, e.message)
             return
 
-        self.write(dict([(u.id, u.serialize()) for u in users]))
+        self.write({u.id: u.serialize() for u in users})
 
     def post(self):
         """HTTP POST method."""

zoe_api/web/request_handler.py

@@ -34,7 +34,7 @@ from zoe_api.exceptions import ZoeAuthException
 log = logging.getLogger(__name__)
 
 
-class JinjaApp(object):
+class JinjaApp:
     """A Jinja2-capable Tornado application."""
     def __init__(self, application, jinja_options=None):
         self.application = application

zoe_master/backends/docker/threads.py

@@ -113,7 +113,7 @@ class DockerStateSynchronizer(threading.Thread):
                     image = {
                         'id': dk_image.attrs['Id'],
                         'size': dk_image.attrs['Size'],
-                        'names': dk_image.tags  # type: list
+                        'names': dk_image.tags
                     }
                     for name in image['names']:
                         if name[-7:] == ':latest':  # add an image with the name without 'latest' to fake Docker image lookup algorithm

zoe_master/backends/kubernetes/api_client.py

@@ -352,7 +352,7 @@ class KubernetesClient:
 
             info = {
                 'service_name': name,
-                'port_forwarding': []  # type: List[Dict]
+                'port_forwarding': []
             }
 
             if 'clusterIP' in srv_info['spec']:

zoe_master/entrypoint.py

@@ -20,7 +20,7 @@
 import logging
 import os
 
-import zoe_lib.config as config
+import zoe_lib.config
 from zoe_lib.state import SQLManager
 import zoe_master.backends.interface
 import zoe_master.scheduler
@@ -35,8 +35,8 @@ LOG_FORMAT = '%(asctime)-15s %(levelname)s %(threadName)s->%(name)s: %(message)s
 
 
 def _check_configuration_sanity():
-    if not os.path.exists(os.path.join(config.get_conf().workspace_base_path, config.get_conf().workspace_deployment_path)):
-        log.error('Workspace base directory does not exist: {}'.format(os.path.join(config.get_conf().workspace_base_path, config.get_conf().workspace_deployment_path)))
+    if not os.path.exists(os.path.join(config.get_conf().workspace_base_path, zoe_lib.config.get_conf().workspace_deployment_path)):
+        log.error('Workspace base directory does not exist: {}'.format(os.path.join(zoe_lib.config.get_conf().workspace_base_path, config.get_conf().workspace_deployment_path)))
         return 1
     return 0
 
@@ -46,8 +46,8 @@ def main(test_conf=None):
     The entrypoint for the zoe-master script.
     :return: int
     """
-    config.load_configuration(test_conf)
-    args = config.get_conf()
+    zoe_lib.config.load_configuration(test_conf)
+    args = zoe_lib.config.get_conf()
 
     log_args = {
         'level': logging.DEBUG if args.debug else logging.INFO,
@@ -81,7 +81,7 @@ def main(test_conf=None):
     log.info("Starting ZMQ API server...")
     api_server = APIManager(metrics, scheduler, state)
 
-    if config.get_conf().gelf_listener != 0:
+    if zoe_lib.config.get_conf().gelf_listener != 0:
         gelf_listener = GELFListener()
     else:
         gelf_listener = None

zoe_master/master_api.py

@@ -20,7 +20,7 @@ import time
 
 import zmq
 
-import zoe_lib.config as config
+import zoe_lib.config
 from zoe_lib.state import SQLManager
 import zoe_master.preprocessing
 from zoe_master.exceptions import ZoeException
@@ -35,7 +35,7 @@ class APIManager:
     def __init__(self, metrics: StatsManager, scheduler: ZoeBaseScheduler, state: SQLManager) -> None:
         self.context = zmq.Context()
         self.zmq_s = self.context.socket(zmq.REP)
-        self.listen_uri = config.get_conf().api_listen_uri
+        self.listen_uri = zoe_lib.config.get_conf().api_listen_uri
         self.zmq_s.bind(self.listen_uri)
         self.debug_has_replied = False
         self.metrics = metrics

zoe_master/scheduler/elastic_scheduler.py

@@ -61,7 +61,7 @@ class ExecutionProgress:
 class ZoeElasticScheduler:
     """The Scheduler class for size-based scheduling. Policy can be "FIFO" or "SIZE"."""
     def __init__(self, state: SQLManager, policy, metrics: StatsManager):
-        if policy != 'FIFO' and policy != 'SIZE' and policy != 'DYNSIZE':
+        if policy not in ('FIFO', 'SIZE', 'DYNSIZE'):
             raise UnsupportedSchedulerPolicyError
         self.metrics = metrics
         self.trigger_semaphore = threading.Semaphore(0)

zoe_master/workspace/filesystem.py

@@ -18,7 +18,7 @@
 import logging
 import os
 
-import zoe_lib.config as config
+import zoe_lib.config
 from zoe_lib.state import VolumeDescriptionHostPath, User
 import zoe_master.workspace.base
 
@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
 class ZoeFSWorkspace(zoe_master.workspace.base.ZoeWorkspaceBase):
     """Filesystem workspace class."""
     def __init__(self):
-        self.base_path = os.path.join(config.get_conf().workspace_base_path, config.get_conf().workspace_deployment_path)
+        self.base_path = os.path.join(zoe_lib.config.get_conf().workspace_base_path, zoe_lib.config.get_conf().workspace_deployment_path)
 
     def exists(self, username):
         """Check if the workspace for user_id exists."""
@@ -53,7 +53,7 @@ class ZoeFSWorkspace(zoe_master.workspace.base.ZoeWorkspaceBase):
         if not self.exists(user.username):
             try:
                 os.makedirs(self.get_path(user.username), 0x700)
-                os.chown(self.get_path(user.username), user.fs_uid, config.get_conf().fs_group_id)
+                os.chown(self.get_path(user.username), user.fs_uid, zoe_lib.config.get_conf().fs_group_id)
             except OSError as e:
                 log.warning("Cannot create user workspace automatically: {}".format(str(e)))
         else: