Merge branch 'devel/platform_status' into 'master'

Various fixes and clean-ups

See merge request !50

CHANGELOG.md

 # Zoe Changelog
 
+## Version 2017.12
+
+* Status page for the administrator
+* More information about authentications in the log output of zoe-api
+
 ## Version 2017.09
 
 * Major web UI redesign

create_db_tables.py

@@ -15,15 +15,22 @@
 
 """Create the DB tables needed by Zoe. This script is used in the CI pipeline to prevent race conditions with zoe-api automatically creating the tables while zoe-master is starting at the same time."""
 
+import sys
 import time
 
 import zoe_lib.config as config
-import zoe_api.db_init
+import zoe_lib.state.sql_manager
 
 config.load_configuration()
 
 print("Warning, this script will delete the database tables for the deployment '{}' before creating new ones".format(config.get_conf().deployment_name))
+print("If you are installing Zoe for the first time, you have nothing to worry about")
 print("Sleeping 5 seconds before continuing, hit CTRL-C to stop and think.")
-time.sleep(5)
 
-zoe_api.db_init.init(force=True)
+try:
+    time.sleep(5)
+except KeyboardInterrupt:
+    print("Aborted.")
+    sys.exit(1)
+
+zoe_lib.state.sql_manager.SQLManager(config.get_conf()).init_db(force=True)

zoe_api/api_endpoint.py

@@ -37,23 +37,23 @@ class APIEndpoint:
     :type master: zoe_api.master_api.APIManager
     :type sql: zoe_lib.sql_manager.SQLManager
     """
-    def __init__(self, master_api, sql_manager):
+    def __init__(self, master_api, sql_manager: zoe_lib.state.SQLManager):
         self.master = master_api
         self.sql = sql_manager
 
-    def execution_by_id(self, uid, role, execution_id) -> zoe_lib.state.sql_manager.Execution:
+    def execution_by_id(self, uid, role, execution_id) -> zoe_lib.state.Execution:
         """Lookup an execution by its ID."""
-        e = self.sql.execution_list(id=execution_id, only_one=True)
+        e = self.sql.executions.select(id=execution_id, only_one=True)
         if e is None:
             raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
-        assert isinstance(e, zoe_lib.state.sql_manager.Execution)
+        assert isinstance(e, zoe_lib.state.Execution)
         if e.user_id != uid and role != 'admin':
             raise zoe_api.exceptions.ZoeAuthException()
         return e
 
     def execution_list(self, uid, role, **filters):
         """Generate a optionally filtered list of executions."""
-        execs = self.sql.execution_list(**filters)
+        execs = self.sql.executions.select(**filters)
         ret = [e for e in execs if e.user_id == uid or role == 'admin']
         return ret
 
@@ -64,7 +64,7 @@ class APIEndpoint:
         except zoe_lib.exceptions.InvalidApplicationDescription as e:
             raise zoe_api.exceptions.ZoeException('Invalid application description: ' + e.message)
 
-    def execution_start(self, uid, role, exec_name, application_description): # pylint: disable=unused-argument
+    def execution_start(self, uid, role, exec_name, application_description):  # pylint: disable=unused-argument
         """Start an execution."""
         try:
             zoe_lib.applications.app_validate(application_description)
@@ -81,7 +81,7 @@ class APIEndpoint:
             if len(running_execs) >= GUEST_QUOTA_MAX_EXECUTIONS:
                 raise zoe_api.exceptions.ZoeException('Guest users cannot run more than one execution at a time, quota exceeded.')
 
-        new_id = self.sql.execution_new(exec_name, uid, application_description)
+        new_id = self.sql.executions.insert(exec_name, uid, application_description)
         success, message = self.master.execution_start(new_id)
         if not success:
             raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up ({}).'.format(message))
@@ -90,8 +90,8 @@ class APIEndpoint:
 
     def execution_terminate(self, uid, role, exec_id):
         """Terminate an execution."""
-        e = self.sql.execution_list(id=exec_id, only_one=True)
-        assert isinstance(e, zoe_lib.state.sql_manager.Execution)
+        e = self.sql.executions.select(id=exec_id, only_one=True)
+        assert isinstance(e, zoe_lib.state.Execution)
         if e is None:
             raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
 
@@ -108,8 +108,8 @@ class APIEndpoint:
         if role != "admin":
             raise zoe_api.exceptions.ZoeAuthException()
 
-        e = self.sql.execution_list(id=exec_id, only_one=True)
-        assert isinstance(e, zoe_lib.state.sql_manager.Execution)
+        e = self.sql.executions.select(id=exec_id, only_one=True)
+        assert isinstance(e, zoe_lib.state.Execution)
         if e is None:
             raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
 
@@ -121,14 +121,14 @@ class APIEndpoint:
 
         status, message = self.master.execution_delete(exec_id)
         if status:
-            self.sql.execution_delete(exec_id)
+            self.sql.executions.delete(exec_id)
             return True, ''
         else:
             raise zoe_api.exceptions.ZoeException(message)
 
-    def service_by_id(self, uid, role, service_id) -> zoe_lib.state.sql_manager.Service:
+    def service_by_id(self, uid, role, service_id) -> zoe_lib.state.Service:
         """Lookup a service by its ID."""
-        service = self.sql.service_list(id=service_id, only_one=True)
+        service = self.sql.services.select(id=service_id, only_one=True)
         if service is None:
             raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
         if service.user_id != uid and role != 'admin':
@@ -137,7 +137,7 @@ class APIEndpoint:
 
     def service_list(self, uid, role, **filters):
         """Generate a optionally filtered list of services."""
-        services = self.sql.service_list(**filters)
+        services = self.sql.services.select(**filters)
         ret = [s for s in services if s.user_id == uid or role == 'admin']
         return ret
 
@@ -145,7 +145,7 @@ class APIEndpoint:
         """Retrieve the logs for the given service.
         If stream is True, a file object is returned, otherwise the log contents as a str object.
         """
-        service = self.sql.service_list(id=service_id, only_one=True)
+        service = self.sql.services.select(id=service_id, only_one=True)
         if service is None:
             raise zoe_api.exceptions.ZoeNotFoundException('No such service')
         if service.user_id != uid and role != 'admin':
@@ -165,7 +165,7 @@ class APIEndpoint:
     def cleanup_dead_executions(self):
         """Terminates all executions with dead "monitor" services."""
         log.debug('Starting dead execution cleanup task')
-        all_execs = self.sql.execution_list(status='running')
+        all_execs = self.sql.executions.select(status='running')
         for execution in all_execs:
             for service in execution.services:
                 if service.description['monitor'] and service.backend_status == service.BACKEND_DIE_STATUS:
@@ -183,7 +183,7 @@ class APIEndpoint:
             services_info.append(self.service_by_id(uid, role, service.id))
             for port in service.description['ports']:
                 port_key = str(port['port_number']) + "/" + port['protocol']
-                backend_port = self.sql.port_list(only_one=True, service_id=service.id, internal_name=port_key)
+                backend_port = self.sql.ports.select(only_one=True, service_id=service.id, internal_name=port_key)
                 if backend_port is not None and backend_port.external_ip is not None:
                     endpoint = port['url_template'].format(**{"ip_port": backend_port.external_ip + ":" + str(backend_port.external_port)})
                     endpoints.append((port['name'], endpoint))

zoe_api/auth/oauth2/postgresql.py

-# Copyright (c) 2017, Quang-Nhat HOANG-XUAN
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-""" Store adapters to read/write data to from/to PostgresSQL. """
-
-import zoe_lib.state
-
-from zoe_lib.config import get_conf
-from oauth2.store import AccessTokenStore, ClientStore
-from oauth2.datatype import AccessToken, Client
-from oauth2.error import AccessTokenNotFound, ClientNotFoundError
-
-class AccessTokenStorePg(AccessTokenStore):
-    """ AccessTokenStore for postgresql  """
-
-    def fetch_by_refresh_token(self, refresh_token):
-        """ get accesstoken from refreshtoken """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        data = sql.fetch_by_refresh_token(refresh_token)
-
-        if data is None:
-            raise AccessTokenNotFound
-
-        return AccessToken(client_id=data["client_id"],
-                           grant_type=data["grant_type"],
-                           token=data["token"],
-                           data=data["data"],
-                           expires_at=data["expires_at"].timestamp(),
-                           refresh_token=data["refresh_token"],
-                           refresh_expires_at=data["refresh_token_expires_at"].timestamp(),
-                           scopes=data["scopes"])
-
-    def delete_refresh_token(self, refresh_token):
-        """
-        Deletes (invalidates) an old refresh token after use
-        :param refresh_token: The refresh token.
-        """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        res = sql.delete_refresh_token(refresh_token)
-        return res
-
-    def get_client_id_by_refresh_token(self, refresh_token):
-        """ get clientID from refreshtoken """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        data = sql.get_client_id_by_refresh_token(refresh_token)
-
-        return data
-
-    def get_client_id_by_access_token(self, access_token):
-        """ get clientID from accesstoken """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        data = sql.get_client_id_by_access_token(access_token)
-
-        return data
-
-    def fetch_existing_token_of_user(self, client_id, grant_type, user_id):
-        """ get accesstoken from userid """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        data = sql.fetch_existing_token_of_user(client_id, grant_type, user_id)
-
-        if data is None:
-            raise AccessTokenNotFound
-
-        return AccessToken(client_id=data["client_id"],
-                           grant_type=data["grant_type"],
-                           token=data["token"],
-                           data=data["data"],
-                           expires_at=data["expires_at"].timestamp(),
-                           refresh_token=data["refresh_token"],
-                           refresh_expires_at=data["refresh_token_expires_at"].timestamp(),
-                           scopes=data["scopes"],
-                           user_id=data["user_id"])
-
-    def save_token(self, access_token):
-        """ save accesstoken """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        sql.save_token(access_token.client_id,
-                       access_token.grant_type,
-                       access_token.token,
-                       access_token.data,
-                       access_token.expires_at,
-                       access_token.refresh_token,
-                       access_token.refresh_expires_at,
-                       access_token.scopes,
-                       access_token.user_id)
-
-        return True
-
-
-class ClientStorePg(ClientStore):
-    """ ClientStore for postgres """
-
-    def save_client(self, identifier, secret, role, redirect_uris, authorized_grants, authorized_response_types):
-        """ save client to db """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        sql.save_client(identifier,
-                        secret,
-                        role,
-                        redirect_uris,
-                        authorized_grants,
-                        authorized_response_types)
-        return True
-
-    def fetch_by_client_id(self, client_id):
-        """ get client by clientid """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        client_data = sql.fetch_by_client_id(client_id)
-
-        client_data_grants = client_data["authorized_grants"].split(':')
-
-        if client_data is None:
-            raise ClientNotFoundError
-
-        return Client(identifier=client_data["identifier"],
-                      secret=client_data["secret"],
-                      redirect_uris=client_data["redirect_uris"],
-                      authorized_grants=client_data_grants,
-                      authorized_response_types=client_data["authorized_response_types"])
-
-    def get_role_by_client_id(self, client_id):
-        """ get client role by clientid """
-        sql = zoe_lib.state.SQLManager(get_conf())
-        client_data = sql.fetch_by_client_id(client_id)
-
-        if client_data is None:
-            raise ClientNotFoundError
-
-        return client_data["role"]

zoe_api/auth/oauth2/tokengenerator.py

-# Copyright (c) 2016, Quang-Nhat HOANG-XUAN
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-""" Token generator for oauth2."""
-
-import hashlib
-import os
-import uuid
-
-class TokenGenerator(object):
-    """
-    Base class of every token generator.
-    """
-    def __init__(self):
-        """
-        Create a new instance of a token generator.
-        """
-        self.expires_in = {}
-        self.refresh_expires_in = 3600
-
-    def create_access_token_data(self, grant_type):
-        """
-        Create data needed by an access token.
-
-        :param grant_type:
-        :type grant_type: str
-
-        :return: A ``dict`` containing he ``access_token`` and the
-                 ``token_type``. If the value of ``TokenGenerator.expires_in``
-                 is larger than 0, a ``refresh_token`` will be generated too.
-        :rtype: dict
-        """
-
-        if grant_type == 'password':
-            self.expires_in['password'] = 36000
-
-        result = {"access_token": self.generate(), "token_type": "Bearer"}
-
-        if self.expires_in.get(grant_type, 0) > 0:
-            result["refresh_token"] = self.generate()
-
-            result["expires_in"] = self.expires_in[grant_type]
-
-        return result
-
-    def generate(self):
-        """
-        Implemented by generators extending this base class.
-
-        :raises NotImplementedError:
-        """
-        raise NotImplementedError
-
-class URandomTokenGenerator(TokenGenerator):
-    """
-    Create a token using ``os.urandom()``.
-    """
-    def __init__(self, length=40):
-        self.token_length = length
-        TokenGenerator.__init__(self)
-
-    def generate(self):
-        """
-        :return: A new token
-        :rtype: str
-        """
-        random_data = os.urandom(100)
-
-        hash_gen = hashlib.new("sha512")
-        hash_gen.update(random_data)
-
-        return hash_gen.hexdigest()[:self.token_length]
-
-class Uuid4(TokenGenerator):
-    """
-    Generate a token using uuid4.
-    """
-    def generate(self):
-        """
-        :return: A new token
-        :rtype: str
-        """
-        return str(uuid.uuid4())

zoe_api/db_init.py

-# Copyright (c) 2016, Daniele Venzano
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Database initialization."""
-
-import psycopg2
-import psycopg2.extras
-
-import zoe_api.exceptions
-from zoe_lib.config import get_conf
-
-SQL_SCHEMA_VERSION = 5  # ---> Increment this value every time the schema changes !!! <---
-
-
-def version_table(cur):
-    """Create the version table."""
-    cur.execute("CREATE TABLE IF NOT EXISTS public.versions (deployment text, version integer)")
-
-
-def schema(cur, deployment_name):
-    """Create the schema for the configured deployment name."""
-    cur.execute("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = %s)", (deployment_name,))
-    if not cur.fetchone()[0]:
-        cur.execute('CREATE SCHEMA {}'.format(deployment_name))
-
-
-def check_schema_version(cur, deployment_name):
-    """Check if the schema version matches this source code version."""
-    cur.execute("SELECT version FROM public.versions WHERE deployment = %s", (deployment_name,))
-    row = cur.fetchone()
-    if row is None:
-        cur.execute("INSERT INTO public.versions (deployment, version) VALUES (%s, %s)", (deployment_name, SQL_SCHEMA_VERSION))
-        schema(cur, deployment_name)
-        return False  # Tables need to be created
-    else:
-        if row[0] == SQL_SCHEMA_VERSION:
-            return True
-        else:
-            raise zoe_api.exceptions.ZoeException('SQL database schema version mismatch: need {}, found {}'.format(SQL_SCHEMA_VERSION, row[0]))
-
-
-def create_tables(cur):
-    """Create the Zoe database tables."""
-    cur.execute('''CREATE TABLE execution (
-        id SERIAL PRIMARY KEY,
-        name TEXT NOT NULL,
-        user_id TEXT NOT NULL,
-        description JSON NOT NULL,
-        status TEXT NOT NULL,
-        execution_manager_id TEXT NULL,
-        time_submit TIMESTAMP NOT NULL,
-        time_start TIMESTAMP NULL,
-        time_end TIMESTAMP NULL,
-        error_message TEXT NULL
-        )''')
-    cur.execute('''CREATE TABLE service (
-        id SERIAL PRIMARY KEY,
-        status TEXT NOT NULL,
-        error_message TEXT NULL DEFAULT NULL,
-        description JSON NOT NULL,
-        execution_id INT REFERENCES execution ON DELETE CASCADE,
-        service_group TEXT NOT NULL,
-        name TEXT NOT NULL,
-        backend_id TEXT NULL DEFAULT NULL,
-        backend_status TEXT NOT NULL DEFAULT 'undefined',
-        backend_host TEXT NULL DEFAULT NULL,
-        ip_address CIDR NULL DEFAULT NULL,
-        essential BOOLEAN NOT NULL DEFAULT FALSE
-        )''')
-    cur.execute('''CREATE TABLE port (
-        id SERIAL PRIMARY KEY,
-        service_id INT REFERENCES service ON DELETE CASCADE,
-        internal_name TEXT NOT NULL,
-        external_ip INET NULL,
-        external_port INT NULL,
-        description JSON NOT NULL
-    )''')
-    #Create oauth_client and oauth_token tables for oAuth2
-    cur.execute('''CREATE TABLE oauth_client (
-        identifier TEXT PRIMARY KEY,
-        secret TEXT,
-        role TEXT,
-        redirect_uris TEXT,
-        authorized_grants TEXT,
-        authorized_response_types TEXT
-        )''')
-    cur.execute('''CREATE TABLE oauth_token (
-        client_id TEXT PRIMARY KEY,
-        grant_type TEXT,
-        token TEXT,
-        data TEXT,
-        expires_at TIMESTAMP,
-        refresh_token TEXT,
-        refresh_token_expires_at TIMESTAMP,
-        scopes TEXT,
-        user_id TEXT
-        )''')
-
-
-def init(force=False):
-    """DB init entrypoint."""
-    dsn = 'dbname=' + get_conf().dbname + \
-        ' user=' + get_conf().dbuser + \
-        ' password=' + get_conf().dbpass + \
-        ' host=' + get_conf().dbhost + \
-        ' port=' + str(get_conf().dbport)
-
-    conn = psycopg2.connect(dsn)
-    cur = conn.cursor()
-
-    version_table(cur)
-    cur.execute('SET search_path TO {},public'.format(get_conf().deployment_name))
-
-    if force:
-        cur.execute("DELETE FROM public.versions WHERE deployment = %s", (get_conf().deployment_name,))
-        cur.execute('DROP SCHEMA IF EXISTS {} CASCADE'.format(get_conf().deployment_name))
-
-    if not check_schema_version(cur, get_conf().deployment_name):
-        create_tables(cur)
-
-    conn.commit()
-    cur.close()
-    conn.close()
-    return

zoe_api/entrypoint.py

@@ -24,7 +24,6 @@ from tornado.web import Application
 
 import zoe_lib.config as config
 import zoe_lib.state
-import zoe_api.db_init
 import zoe_api.api_endpoint
 import zoe_api.rest_api
 import zoe_api.master_api
@@ -57,10 +56,10 @@ def zoe_web_main() -> int:
         log.error("LDAP authentication requested, but 'pyldap' module not installed.")
         return 1
 
-    zoe_api.db_init.init()
+    sql_manager = zoe_lib.state.SQLManager(config.get_conf())
+    sql_manager.init_db()
 
     master_api = zoe_api.master_api.APIManager()
-    sql_manager = zoe_lib.state.SQLManager(config.get_conf())
     api_endpoint = zoe_api.api_endpoint.APIEndpoint(master_api, sql_manager)
 
     app_settings = {

zoe_api/rest_api/__init__.py

@@ -25,7 +25,6 @@ from zoe_api.rest_api.userinfo import UserInfoAPI
 from zoe_api.rest_api.service import ServiceAPI, ServiceLogsAPI
 from zoe_api.rest_api.discovery import DiscoveryAPI
 from zoe_api.rest_api.statistics import SchedulerStatsAPI
-from zoe_api.rest_api.oauth import OAuthGetAPI, OAuthRevokeAPI
 from zoe_api.rest_api.login import LoginAPI
 from zoe_api.rest_api.validation import ZAppValidateAPI
 
@@ -56,10 +55,7 @@ def api_init(api_endpoint) -> List[tornado.web.URLSpec]:
 
         tornado.web.url(API_PATH + r'/discovery/by_group/([0-9]+)/([a-z0-9A-Z\-]+)', DiscoveryAPI, route_args),
 
-        tornado.web.url(API_PATH + r'/statistics/scheduler', SchedulerStatsAPI, route_args),
-
-        tornado.web.url(API_PATH + r'/oauth/token', OAuthGetAPI, route_args),
-        tornado.web.url(API_PATH + r'/oauth/revoke/([a-z0-9A-Z\-]+)', OAuthRevokeAPI, route_args)
+        tornado.web.url(API_PATH + r'/statistics/scheduler', SchedulerStatsAPI, route_args)
     ]
 
     return api_routes

zoe_api/rest_api/oauth.py

-# Copyright (c) 2016, Daniele Venzano
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""The oAuth2 API endpoints."""
-
-import logging
-import json
-import psycopg2
-
-from tornado.web import RequestHandler
-
-import oauth2.grant
-
-from zoe_api.rest_api.utils import catch_exceptions, get_auth
-from zoe_api.rest_api.oauth_utils import auth_controller, client_store, token_store
-from zoe_api.rest_api.utils import manage_cors_headers
-from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
-
-log = logging.getLogger(__name__)
-
-"""
-
-Example of using:
-
-*To request a new token of type: