Commit 06d52137 authored by Daniele Venzano's avatar Daniele Venzano 🏇
Browse files

Merge branch 'devel/platform_status' into 'master'

Various fixes and clean-ups

See merge request !50
parents 1548a635 a0adc5ee
# Zoe Changelog
## Version 2017.12
* Status page for the administrator
* More information about authentications in the log output of zoe-api
## Version 2017.09
* Major web UI redesign
......
......@@ -15,15 +15,22 @@
"""Create the DB tables needed by Zoe. This script is used in the CI pipeline to prevent race conditions with zoe-api automatically creating the tables while zoe-master is starting at the same time."""
import sys
import time
import zoe_lib.config as config
import zoe_api.db_init
import zoe_lib.state.sql_manager
config.load_configuration()
print("Warning, this script will delete the database tables for the deployment '{}' before creating new ones".format(config.get_conf().deployment_name))
print("If you are installing Zoe for the first time, you have nothing to worry about")
print("Sleeping 5 seconds before continuing, hit CTRL-C to stop and think.")
time.sleep(5)
zoe_api.db_init.init(force=True)
try:
time.sleep(5)
except KeyboardInterrupt:
print("Aborted.")
sys.exit(1)
zoe_lib.state.sql_manager.SQLManager(config.get_conf()).init_db(force=True)
......@@ -37,23 +37,23 @@ class APIEndpoint:
:type master: zoe_api.master_api.APIManager
:type sql: zoe_lib.sql_manager.SQLManager
"""
def __init__(self, master_api, sql_manager):
def __init__(self, master_api, sql_manager: zoe_lib.state.SQLManager):
self.master = master_api
self.sql = sql_manager
def execution_by_id(self, uid, role, execution_id) -> zoe_lib.state.sql_manager.Execution:
def execution_by_id(self, uid, role, execution_id) -> zoe_lib.state.Execution:
"""Lookup an execution by its ID."""
e = self.sql.execution_list(id=execution_id, only_one=True)
e = self.sql.executions.select(id=execution_id, only_one=True)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
assert isinstance(e, zoe_lib.state.sql_manager.Execution)
assert isinstance(e, zoe_lib.state.Execution)
if e.user_id != uid and role != 'admin':
raise zoe_api.exceptions.ZoeAuthException()
return e
def execution_list(self, uid, role, **filters):
"""Generate a optionally filtered list of executions."""
execs = self.sql.execution_list(**filters)
execs = self.sql.executions.select(**filters)
ret = [e for e in execs if e.user_id == uid or role == 'admin']
return ret
......@@ -64,7 +64,7 @@ class APIEndpoint:
except zoe_lib.exceptions.InvalidApplicationDescription as e:
raise zoe_api.exceptions.ZoeException('Invalid application description: ' + e.message)
def execution_start(self, uid, role, exec_name, application_description): # pylint: disable=unused-argument
def execution_start(self, uid, role, exec_name, application_description): # pylint: disable=unused-argument
"""Start an execution."""
try:
zoe_lib.applications.app_validate(application_description)
......@@ -81,7 +81,7 @@ class APIEndpoint:
if len(running_execs) >= GUEST_QUOTA_MAX_EXECUTIONS:
raise zoe_api.exceptions.ZoeException('Guest users cannot run more than one execution at a time, quota exceeded.')
new_id = self.sql.execution_new(exec_name, uid, application_description)
new_id = self.sql.executions.insert(exec_name, uid, application_description)
success, message = self.master.execution_start(new_id)
if not success:
raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up ({}).'.format(message))
......@@ -90,8 +90,8 @@ class APIEndpoint:
def execution_terminate(self, uid, role, exec_id):
"""Terminate an execution."""
e = self.sql.execution_list(id=exec_id, only_one=True)
assert isinstance(e, zoe_lib.state.sql_manager.Execution)
e = self.sql.executions.select(id=exec_id, only_one=True)
assert isinstance(e, zoe_lib.state.Execution)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
......@@ -108,8 +108,8 @@ class APIEndpoint:
if role != "admin":
raise zoe_api.exceptions.ZoeAuthException()
e = self.sql.execution_list(id=exec_id, only_one=True)
assert isinstance(e, zoe_lib.state.sql_manager.Execution)
e = self.sql.executions.select(id=exec_id, only_one=True)
assert isinstance(e, zoe_lib.state.Execution)
if e is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
......@@ -121,14 +121,14 @@ class APIEndpoint:
status, message = self.master.execution_delete(exec_id)
if status:
self.sql.execution_delete(exec_id)
self.sql.executions.delete(exec_id)
return True, ''
else:
raise zoe_api.exceptions.ZoeException(message)
def service_by_id(self, uid, role, service_id) -> zoe_lib.state.sql_manager.Service:
def service_by_id(self, uid, role, service_id) -> zoe_lib.state.Service:
"""Lookup a service by its ID."""
service = self.sql.service_list(id=service_id, only_one=True)
service = self.sql.services.select(id=service_id, only_one=True)
if service is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
if service.user_id != uid and role != 'admin':
......@@ -137,7 +137,7 @@ class APIEndpoint:
def service_list(self, uid, role, **filters):
"""Generate a optionally filtered list of services."""
services = self.sql.service_list(**filters)
services = self.sql.services.select(**filters)
ret = [s for s in services if s.user_id == uid or role == 'admin']
return ret
......@@ -145,7 +145,7 @@ class APIEndpoint:
"""Retrieve the logs for the given service.
If stream is True, a file object is returned, otherwise the log contents as a str object.
"""
service = self.sql.service_list(id=service_id, only_one=True)
service = self.sql.services.select(id=service_id, only_one=True)
if service is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such service')
if service.user_id != uid and role != 'admin':
......@@ -165,7 +165,7 @@ class APIEndpoint:
def cleanup_dead_executions(self):
"""Terminates all executions with dead "monitor" services."""
log.debug('Starting dead execution cleanup task')
all_execs = self.sql.execution_list(status='running')
all_execs = self.sql.executions.select(status='running')
for execution in all_execs:
for service in execution.services:
if service.description['monitor'] and service.backend_status == service.BACKEND_DIE_STATUS:
......@@ -183,7 +183,7 @@ class APIEndpoint:
services_info.append(self.service_by_id(uid, role, service.id))
for port in service.description['ports']:
port_key = str(port['port_number']) + "/" + port['protocol']
backend_port = self.sql.port_list(only_one=True, service_id=service.id, internal_name=port_key)
backend_port = self.sql.ports.select(only_one=True, service_id=service.id, internal_name=port_key)
if backend_port is not None and backend_port.external_ip is not None:
endpoint = port['url_template'].format(**{"ip_port": backend_port.external_ip + ":" + str(backend_port.external_port)})
endpoints.append((port['name'], endpoint))
......
# Copyright (c) 2017, Quang-Nhat HOANG-XUAN
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Store adapters to read/write data to from/to PostgresSQL. """
import zoe_lib.state
from zoe_lib.config import get_conf
from oauth2.store import AccessTokenStore, ClientStore
from oauth2.datatype import AccessToken, Client
from oauth2.error import AccessTokenNotFound, ClientNotFoundError
class AccessTokenStorePg(AccessTokenStore):
""" AccessTokenStore for postgresql """
def fetch_by_refresh_token(self, refresh_token):
""" get accesstoken from refreshtoken """
sql = zoe_lib.state.SQLManager(get_conf())
data = sql.fetch_by_refresh_token(refresh_token)
if data is None:
raise AccessTokenNotFound
return AccessToken(client_id=data["client_id"],
grant_type=data["grant_type"],
token=data["token"],
data=data["data"],
expires_at=data["expires_at"].timestamp(),
refresh_token=data["refresh_token"],
refresh_expires_at=data["refresh_token_expires_at"].timestamp(),
scopes=data["scopes"])
def delete_refresh_token(self, refresh_token):
"""
Deletes (invalidates) an old refresh token after use
:param refresh_token: The refresh token.
"""
sql = zoe_lib.state.SQLManager(get_conf())
res = sql.delete_refresh_token(refresh_token)
return res
def get_client_id_by_refresh_token(self, refresh_token):
""" get clientID from refreshtoken """
sql = zoe_lib.state.SQLManager(get_conf())
data = sql.get_client_id_by_refresh_token(refresh_token)
return data
def get_client_id_by_access_token(self, access_token):
""" get clientID from accesstoken """
sql = zoe_lib.state.SQLManager(get_conf())
data = sql.get_client_id_by_access_token(access_token)
return data
def fetch_existing_token_of_user(self, client_id, grant_type, user_id):
""" get accesstoken from userid """
sql = zoe_lib.state.SQLManager(get_conf())
data = sql.fetch_existing_token_of_user(client_id, grant_type, user_id)
if data is None:
raise AccessTokenNotFound
return AccessToken(client_id=data["client_id"],
grant_type=data["grant_type"],
token=data["token"],
data=data["data"],
expires_at=data["expires_at"].timestamp(),
refresh_token=data["refresh_token"],
refresh_expires_at=data["refresh_token_expires_at"].timestamp(),
scopes=data["scopes"],
user_id=data["user_id"])
def save_token(self, access_token):
""" save accesstoken """
sql = zoe_lib.state.SQLManager(get_conf())
sql.save_token(access_token.client_id,
access_token.grant_type,
access_token.token,
access_token.data,
access_token.expires_at,
access_token.refresh_token,
access_token.refresh_expires_at,
access_token.scopes,
access_token.user_id)
return True
class ClientStorePg(ClientStore):
""" ClientStore for postgres """
def save_client(self, identifier, secret, role, redirect_uris, authorized_grants, authorized_response_types):
""" save client to db """
sql = zoe_lib.state.SQLManager(get_conf())
sql.save_client(identifier,
secret,
role,
redirect_uris,
authorized_grants,
authorized_response_types)
return True
def fetch_by_client_id(self, client_id):
""" get client by clientid """
sql = zoe_lib.state.SQLManager(get_conf())
client_data = sql.fetch_by_client_id(client_id)
client_data_grants = client_data["authorized_grants"].split(':')
if client_data is None:
raise ClientNotFoundError
return Client(identifier=client_data["identifier"],
secret=client_data["secret"],
redirect_uris=client_data["redirect_uris"],
authorized_grants=client_data_grants,
authorized_response_types=client_data["authorized_response_types"])
def get_role_by_client_id(self, client_id):
""" get client role by clientid """
sql = zoe_lib.state.SQLManager(get_conf())
client_data = sql.fetch_by_client_id(client_id)
if client_data is None:
raise ClientNotFoundError
return client_data["role"]
# Copyright (c) 2016, Quang-Nhat HOANG-XUAN
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Token generator for oauth2."""
import hashlib
import os
import uuid
class TokenGenerator(object):
"""
Base class of every token generator.
"""
def __init__(self):
"""
Create a new instance of a token generator.
"""
self.expires_in = {}
self.refresh_expires_in = 3600
def create_access_token_data(self, grant_type):
"""
Create data needed by an access token.
:param grant_type:
:type grant_type: str
:return: A ``dict`` containing he ``access_token`` and the
``token_type``. If the value of ``TokenGenerator.expires_in``
is larger than 0, a ``refresh_token`` will be generated too.
:rtype: dict
"""
if grant_type == 'password':
self.expires_in['password'] = 36000
result = {"access_token": self.generate(), "token_type": "Bearer"}
if self.expires_in.get(grant_type, 0) > 0:
result["refresh_token"] = self.generate()
result["expires_in"] = self.expires_in[grant_type]
return result
def generate(self):
"""
Implemented by generators extending this base class.
:raises NotImplementedError:
"""
raise NotImplementedError
class URandomTokenGenerator(TokenGenerator):
"""
Create a token using ``os.urandom()``.
"""
def __init__(self, length=40):
self.token_length = length
TokenGenerator.__init__(self)
def generate(self):
"""
:return: A new token
:rtype: str
"""
random_data = os.urandom(100)
hash_gen = hashlib.new("sha512")
hash_gen.update(random_data)
return hash_gen.hexdigest()[:self.token_length]
class Uuid4(TokenGenerator):
"""
Generate a token using uuid4.
"""
def generate(self):
"""
:return: A new token
:rtype: str
"""
return str(uuid.uuid4())
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database initialization."""
import psycopg2
import psycopg2.extras
import zoe_api.exceptions
from zoe_lib.config import get_conf
SQL_SCHEMA_VERSION = 5 # ---> Increment this value every time the schema changes !!! <---
def version_table(cur):
"""Create the version table."""
cur.execute("CREATE TABLE IF NOT EXISTS public.versions (deployment text, version integer)")
def schema(cur, deployment_name):
"""Create the schema for the configured deployment name."""
cur.execute("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = %s)", (deployment_name,))
if not cur.fetchone()[0]:
cur.execute('CREATE SCHEMA {}'.format(deployment_name))
def check_schema_version(cur, deployment_name):
"""Check if the schema version matches this source code version."""
cur.execute("SELECT version FROM public.versions WHERE deployment = %s", (deployment_name,))
row = cur.fetchone()
if row is None:
cur.execute("INSERT INTO public.versions (deployment, version) VALUES (%s, %s)", (deployment_name, SQL_SCHEMA_VERSION))
schema(cur, deployment_name)
return False # Tables need to be created
else:
if row[0] == SQL_SCHEMA_VERSION:
return True
else:
raise zoe_api.exceptions.ZoeException('SQL database schema version mismatch: need {}, found {}'.format(SQL_SCHEMA_VERSION, row[0]))
def create_tables(cur):
"""Create the Zoe database tables."""
cur.execute('''CREATE TABLE execution (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
user_id TEXT NOT NULL,
description JSON NOT NULL,
status TEXT NOT NULL,
execution_manager_id TEXT NULL,
time_submit TIMESTAMP NOT NULL,
time_start TIMESTAMP NULL,
time_end TIMESTAMP NULL,
error_message TEXT NULL
)''')
cur.execute('''CREATE TABLE service (
id SERIAL PRIMARY KEY,
status TEXT NOT NULL,
error_message TEXT NULL DEFAULT NULL,
description JSON NOT NULL,
execution_id INT REFERENCES execution ON DELETE CASCADE,
service_group TEXT NOT NULL,
name TEXT NOT NULL,
backend_id TEXT NULL DEFAULT NULL,
backend_status TEXT NOT NULL DEFAULT 'undefined',
backend_host TEXT NULL DEFAULT NULL,
ip_address CIDR NULL DEFAULT NULL,
essential BOOLEAN NOT NULL DEFAULT FALSE
)''')
cur.execute('''CREATE TABLE port (
id SERIAL PRIMARY KEY,
service_id INT REFERENCES service ON DELETE CASCADE,
internal_name TEXT NOT NULL,
external_ip INET NULL,
external_port INT NULL,
description JSON NOT NULL
)''')
#Create oauth_client and oauth_token tables for oAuth2
cur.execute('''CREATE TABLE oauth_client (
identifier TEXT PRIMARY KEY,
secret TEXT,
role TEXT,
redirect_uris TEXT,
authorized_grants TEXT,
authorized_response_types TEXT
)''')
cur.execute('''CREATE TABLE oauth_token (
client_id TEXT PRIMARY KEY,
grant_type TEXT,
token TEXT,
data TEXT,
expires_at TIMESTAMP,
refresh_token TEXT,
refresh_token_expires_at TIMESTAMP,
scopes TEXT,
user_id TEXT
)''')
def init(force=False):
"""DB init entrypoint."""
dsn = 'dbname=' + get_conf().dbname + \
' user=' + get_conf().dbuser + \
' password=' + get_conf().dbpass + \
' host=' + get_conf().dbhost + \
' port=' + str(get_conf().dbport)
conn = psycopg2.connect(dsn)
cur = conn.cursor()
version_table(cur)
cur.execute('SET search_path TO {},public'.format(get_conf().deployment_name))
if force:
cur.execute("DELETE FROM public.versions WHERE deployment = %s", (get_conf().deployment_name,))
cur.execute('DROP SCHEMA IF EXISTS {} CASCADE'.format(get_conf().deployment_name))
if not check_schema_version(cur, get_conf().deployment_name):
create_tables(cur)
conn.commit()
cur.close()
conn.close()
return
......@@ -24,7 +24,6 @@ from tornado.web import Application
import zoe_lib.config as config
import zoe_lib.state
import zoe_api.db_init
import zoe_api.api_endpoint
import zoe_api.rest_api
import zoe_api.master_api
......@@ -57,10 +56,10 @@ def zoe_web_main() -> int:
log.error("LDAP authentication requested, but 'pyldap' module not installed.")
return 1
zoe_api.db_init.init()
sql_manager = zoe_lib.state.SQLManager(config.get_conf())
sql_manager.init_db()
master_api = zoe_api.master_api.APIManager()
sql_manager = zoe_lib.state.SQLManager(config.get_conf())
api_endpoint = zoe_api.api_endpoint.APIEndpoint(master_api, sql_manager)
app_settings = {
......
......@@ -25,7 +25,6 @@ from zoe_api.rest_api.userinfo import UserInfoAPI
from zoe_api.rest_api.service import ServiceAPI, ServiceLogsAPI
from zoe_api.rest_api.discovery import DiscoveryAPI
from zoe_api.rest_api.statistics import SchedulerStatsAPI
from zoe_api.rest_api.oauth import OAuthGetAPI, OAuthRevokeAPI
from zoe_api.rest_api.login import LoginAPI
from zoe_api.rest_api.validation import ZAppValidateAPI
......@@ -56,10 +55,7 @@ def api_init(api_endpoint) -> List[tornado.web.URLSpec]:
tornado.web.url(API_PATH + r'/discovery/by_group/([0-9]+)/([a-z0-9A-Z\-]+)', DiscoveryAPI, route_args),
tornado.web.url(API_PATH + r'/statistics/scheduler', SchedulerStatsAPI, route_args),
tornado.web.url(API_PATH + r'/oauth/token', OAuthGetAPI, route_args),
tornado.web.url(API_PATH + r'/oauth/revoke/([a-z0-9A-Z\-]+)', OAuthRevokeAPI, route_args)
tornado.web.url(API_PATH + r'/statistics/scheduler', SchedulerStatsAPI, route_args)
]
return api_routes
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The oAuth2 API endpoints."""
import logging
import json
import psycopg2
from tornado.web import RequestHandler
import oauth2.grant
from zoe_api.rest_api.utils import catch_exceptions, get_auth
from zoe_api.rest_api.oauth_utils import auth_controller, client_store, token_store
from zoe_api.rest_api.utils import manage_cors_headers
from zoe_api.api_endpoint import APIEndpoint # pylint: disable=unused-import
log = logging.getLogger(__name__)
"""
Example of using:
*To request a new token of type: