Commit 4d54e953 authored by Daniele Venzano's avatar Daniele Venzano

Rename zoe-web in zoe-api

parent 9cd2c8cf
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_web.entrypoint import zoe_web_main
from zoe_api.entrypoint import zoe_web_main
if __name__ == "__main__":
zoe_web_main()
......@@ -16,31 +16,31 @@
import logging
import re
from zoe_web.config import get_conf
import zoe_web.master_api
from zoe_api.config import get_conf
import zoe_api.master_api
import zoe_lib.sql_manager
import zoe_lib.applications
import zoe_lib.exceptions
import zoe_web.exceptions
import zoe_api.exceptions
log = logging.getLogger(__name__)
class APIEndpoint:
"""
:type master: zoe_web.master_api.APIManager
:type master: zoe_api.master_api.APIManager
:type sql: zoe_lib.sql_manager.SQLManager
"""
def __init__(self):
self.master = zoe_web.master_api.APIManager()
self.master = zoe_api.master_api.APIManager()
self.sql = zoe_lib.sql_manager.SQLManager(get_conf())
def execution_by_id(self, uid, role, execution_id):
e = self.sql.execution_list(id=execution_id, only_one=True)
if e is None:
raise zoe_web.exceptions.ZoeNotFoundException('No such execution')
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
if e.user_id != uid and role != 'admin':
raise zoe_web.exceptions.ZoeAuthException()
raise zoe_api.exceptions.ZoeAuthException()
return e
def execution_list(self, uid, role, **filters):
......@@ -52,28 +52,40 @@ class APIEndpoint:
try:
zoe_lib.applications.app_validate(application_description)
except zoe_lib.exceptions.InvalidApplicationDescription as e:
raise zoe_web.exceptions.ZoeException('Invalid application description: ' + e.message)
raise zoe_api.exceptions.ZoeException('Invalid application description: ' + e.message)
if 3 > len(exec_name) > 128:
raise zoe_web.exceptions.ZoeException("Execution name must be between 4 and 128 characters long")
raise zoe_api.exceptions.ZoeException("Execution name must be between 4 and 128 characters long")
if not re.match(r'^[a-zA-Z0-9\-]+$', exec_name):
raise zoe_web.exceptions.ZoeException("Execution name can contain only letters, numbers and dashes. '{}' is not valid.".format(exec_name))
raise zoe_api.exceptions.ZoeException("Execution name can contain only letters, numbers and dashes. '{}' is not valid.".format(exec_name))
new_id = self.sql.execution_new(exec_name, uid, application_description)
success, message = self.master.execution_start(new_id)
if not success:
raise zoe_web.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up.')
raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up.')
return new_id
def execution_terminate(self, uid, role, exec_id):
e = self.sql.execution_list(id=exec_id, only_one=True)
if e is None:
raise zoe_web.exceptions.ZoeNotFoundException('No such execution')
raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
if e.user_id != uid and role != 'admin':
raise zoe_web.exceptions.ZoeAuthException()
raise zoe_api.exceptions.ZoeAuthException()
if e.is_active():
return self.master.execution_terminate(exec_id)
else:
raise zoe_web.exceptions.ZoeException('Execution is not running')
raise zoe_api.exceptions.ZoeException('Execution is not running')
def service_inspect(self, service):
return self.master.service_inspect(service.id)
def retry_submit_error_executions(self):
waiting_execs = self.sql.execution_list(status=zoe_lib.sql_manager.Execution.SUBMIT_STATUS)
if waiting_execs is None or len(waiting_execs) == 0:
return
e = waiting_execs.pop(0)
success, message = self.master.execution_start(e.id)
if not success:
log.warning('Zoe Master unavailable, execution {} still waiting'.format(e.id))
......@@ -16,13 +16,13 @@ import logging
import ldap
import zoe_web.auth.base
from zoe_web.config import get_conf
import zoe_api.auth.base
from zoe_api.config import get_conf
log = logging.getLogger(__name__)
class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
class LDAPAuthenticator(zoe_api.auth.base.BaseAuthenticator):
def __init__(self):
self.connection = ldap.initialize(get_conf().ldap_server_uri)
self.base_dn = get_conf().ldap_base_dn
......@@ -31,13 +31,13 @@ class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
def auth(self, username, password):
search_filter = "uid=" + username
uid_number = None
uid = None
role = 'guest'
try:
self.connection.bind_s(self.bind_user, self.bind_password)
result = self.connection.search_s(self.base_dn, ldap.SCOPE_SUBTREE, search_filter)
user_dict = result[0][1]
uid_number = int(user_dict['uidNumber'][0])
uid = username
gid_numbers = [int(x) for x in user_dict['gidNumber']]
if get_conf().ldap_admin_gid in gid_numbers:
role = 'admin'
......@@ -52,4 +52,4 @@ class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
log.exception("LDAP exception")
finally:
self.connection.unbind_s()
return uid_number, role
return uid, role
......@@ -18,8 +18,8 @@ from zoe_lib.configargparse import ArgumentParser, Namespace
api_endpoint = None # singleton
config_paths = [
'zoe-web.conf',
'/etc/zoe/zoe-web.conf'
'zoe-api.conf',
'/etc/zoe/zoe-api.conf'
]
_conf = None
......
......@@ -17,7 +17,7 @@ import psycopg2
import psycopg2.extras
import zoe_lib.exceptions
from zoe_web.config import get_conf
from zoe_api.config import get_conf
SQL_SCHEMA_VERSION = 0 # ---> Increment this value every time the schema changes !!! <---
......@@ -50,7 +50,7 @@ def create_tables(cur):
cur.execute('''CREATE TABLE execution (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
user_id INTEGER,
user_id TEXT NOT NULL,
description JSON NOT NULL,
status TEXT NOT NULL,
execution_manager_id TEXT NULL,
......@@ -61,10 +61,13 @@ def create_tables(cur):
)''')
cur.execute('''CREATE TABLE service (
id SERIAL PRIMARY KEY,
execution_id INT REFERENCES execution,
name TEXT NOT NULL ,
status TEXT NOT NULL,
stage TEXT,
error TEXT NULL,
execution_manager_id TEXT
error_message TEXT NULL DEFAULT NULL,
description JSON NOT NULL,
service_group TEXT NOT NULL,
docker_id TEXT NULL DEFAULT NULL
)''')
......
......@@ -18,15 +18,15 @@ import logging
from flask import Flask
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.ioloop import IOLoop, PeriodicCallback
import zoe_web.config as config
import zoe_web.db_init
import zoe_web.api_endpoint
import zoe_web.rest_api
import zoe_web.web
import zoe_api.config as config
import zoe_api.db_init
import zoe_api.api_endpoint
import zoe_api.rest_api
import zoe_api.web
log = logging.getLogger("zoe_web")
log = logging.getLogger("zoe_api")
LOG_FORMAT = '%(asctime)-15s %(levelname)s %(name)s (%(threadName)s): %(message)s'
......@@ -48,16 +48,20 @@ def zoe_web_main() -> int:
app = Flask(__name__, static_url_path='/does-not-exist')
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
app.register_blueprint(zoe_web.rest_api.api_init())
app.register_blueprint(zoe_web.web.web_init())
app.register_blueprint(zoe_api.rest_api.api_init())
app.register_blueprint(zoe_api.web.web_init())
zoe_web.db_init.init()
zoe_api.db_init.init()
config.api_endpoint = zoe_web.api_endpoint.APIEndpoint()
config.api_endpoint = zoe_api.api_endpoint.APIEndpoint()
http_server = HTTPServer(WSGIContainer(app))
http_server.listen(args.listen_port, args.listen_address)
ioloop = IOLoop.instance()
retry_cb = PeriodicCallback(config.api_endpoint.retry_submit_error_executions, 30000)
retry_cb.start()
try:
ioloop.start()
except KeyboardInterrupt:
......
......@@ -17,14 +17,14 @@ import logging
import zmq
import zoe_web.config as config
import zoe_api.config as config
log = logging.getLogger(__name__)
class APIManager:
REQUEST_TIMEOUT = 2500
REQUEST_RETRIES = 3
REQUEST_RETRIES = 1
def __init__(self):
self.context = zmq.Context(1)
......@@ -84,3 +84,10 @@ class APIManager:
'exec_id': exec_id
}
return self._request_reply(msg)
def service_inspect(self, service_id):
msg = {
'command': 'service_inspect',
'service_id': service_id
}
return self._request_reply(msg)
......@@ -17,12 +17,12 @@ import sys
from flask import Blueprint
from flask_restful import Api
from zoe_web.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
from zoe_web.rest_api.info import InfoAPI
from zoe_web.rest_api.service import ServiceAPI
from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
from zoe_api.rest_api.info import InfoAPI
from zoe_api.rest_api.service import ServiceAPI
from zoe_lib.version import ZOE_API_VERSION
from zoe_web.rest_api.query import QueryAPI
from zoe_api.rest_api.query import QueryAPI
API_PATH = '/api/' + ZOE_API_VERSION
......
......@@ -22,10 +22,10 @@ import zoe_lib.exceptions
import zoe_lib.applications
import zoe_lib.sql_manager
from zoe_web.rest_api.utils import catch_exceptions, get_auth
import zoe_web.exceptions
import zoe_web.config as config
import zoe_web.api_endpoint
from zoe_api.rest_api.utils import catch_exceptions, get_auth
import zoe_api.exceptions
import zoe_api.config as config
import zoe_api.api_endpoint
class ExecutionAPI(Resource):
......@@ -33,7 +33,7 @@ class ExecutionAPI(Resource):
def get(self, execution_id):
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
e = config.api_endpoint.execution_by_id(uid, role, execution_id)
return e.serialize()
......@@ -48,10 +48,10 @@ class ExecutionAPI(Resource):
"""
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
success, message = config.api_endpoint.execution_terminate(uid, role, execution_id)
if not success:
raise zoe_web.exceptions.ZoeRestAPIException(message, 400)
raise zoe_api.exceptions.ZoeRestAPIException(message, 400)
return '', 204
......@@ -66,7 +66,7 @@ class ExecutionCollectionAPI(Resource):
"""
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
execs = config.api_endpoint.execution_list(uid, role)
return [e.serialize() for e in execs]
......@@ -78,12 +78,12 @@ class ExecutionCollectionAPI(Resource):
"""
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
try:
data = request.get_json()
except BadRequest:
raise zoe_web.exceptions.ZoeRestAPIException('Error decoding JSON data')
raise zoe_api.exceptions.ZoeRestAPIException('Error decoding JSON data')
application_description = data['application']
exec_name = data['name']
......
......@@ -17,7 +17,7 @@ from flask_restful import Resource
from zoe_lib.version import ZOE_API_VERSION, ZOE_APPLICATION_FORMAT_VERSION, ZOE_VERSION
from zoe_master.config import get_conf
from zoe_web.rest_api.utils import catch_exceptions
from zoe_api.rest_api.utils import catch_exceptions
class InfoAPI(Resource):
......
......@@ -16,17 +16,17 @@
from flask_restful import Resource, request
from werkzeug.exceptions import BadRequest
from zoe_web.exceptions import ZoeRestAPIException
import zoe_web.config as config
import zoe_web.api_endpoint
from zoe_web.rest_api.utils import catch_exceptions, get_auth
from zoe_api.exceptions import ZoeRestAPIException
import zoe_api.config as config
import zoe_api.api_endpoint
from zoe_api.rest_api.utils import catch_exceptions, get_auth
class QueryAPI(Resource):
@catch_exceptions
def post(self):
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
try:
data = request.get_json()
......
......@@ -18,9 +18,8 @@ import time
from flask_restful import Resource, request
from zoe_lib.exceptions import ZoeRestAPIException
from zoe_web.config import singletons
from zoe_web.rest_api.utils import catch_exceptions
from zoe_api.exceptions import ZoeRestAPIException
from zoe_api.rest_api.utils import catch_exceptions
log = logging.getLogger(__name__)
......
......@@ -15,8 +15,8 @@
import logging
from zoe_web.exceptions import ZoeRestAPIException, ZoeNotFoundException, ZoeAuthException, ZoeException
from zoe_web.auth.ldap import LDAPAuthenticator
from zoe_api.exceptions import ZoeRestAPIException, ZoeNotFoundException, ZoeAuthException, ZoeException
from zoe_api.auth.ldap import LDAPAuthenticator
log = logging.getLogger(__name__)
......
......@@ -14,8 +14,8 @@
# limitations under the License.
from flask import Blueprint
import zoe_web.web.start
import zoe_web.web.executions
import zoe_api.web.start
import zoe_api.web.executions
from zoe_lib.version import ZOE_API_VERSION, ZOE_VERSION
......@@ -25,14 +25,14 @@ def web_init() -> Blueprint:
web_bp.context_processor(inject_version)
web_bp.add_url_rule('/', 'index', zoe_web.web.start.index)
web_bp.add_url_rule('/user', 'home_user', zoe_web.web.start.home_user)
web_bp.add_url_rule('/', 'index', zoe_api.web.start.index)
web_bp.add_url_rule('/user', 'home_user', zoe_api.web.start.home_user)
web_bp.add_url_rule('/executions/new', 'execution_define', zoe_web.web.executions.execution_define)
web_bp.add_url_rule('/executions/start', 'execution_start', zoe_web.web.executions.execution_start, methods=['POST'])
web_bp.add_url_rule('/executions/restart/<int:execution_id>', 'execution_restart', zoe_web.web.executions.execution_restart)
web_bp.add_url_rule('/executions/terminate/<int:execution_id>', 'execution_terminate', zoe_web.web.executions.execution_terminate)
web_bp.add_url_rule('/executions/inspect/<int:execution_id>', 'execution_inspect', zoe_web.web.executions.execution_inspect)
web_bp.add_url_rule('/executions/new', 'execution_define', zoe_api.web.executions.execution_define)
web_bp.add_url_rule('/executions/start', 'execution_start', zoe_api.web.executions.execution_start, methods=['POST'])
web_bp.add_url_rule('/executions/restart/<int:execution_id>', 'execution_restart', zoe_api.web.executions.execution_restart)
web_bp.add_url_rule('/executions/terminate/<int:execution_id>', 'execution_terminate', zoe_api.web.executions.execution_terminate)
web_bp.add_url_rule('/executions/inspect/<int:execution_id>', 'execution_inspect', zoe_api.web.executions.execution_inspect)
return web_bp
......
......@@ -17,10 +17,10 @@ import json
from flask import render_template, request, redirect, url_for
from zoe_web.web.utils import get_auth, catch_exceptions
import zoe_web.config as config
import zoe_web.api_endpoint
import zoe_web.exceptions
from zoe_api.web.utils import get_auth, catch_exceptions
import zoe_api.config as config
import zoe_api.api_endpoint
import zoe_api.exceptions
@catch_exceptions
......@@ -33,7 +33,7 @@ def execution_define():
@catch_exceptions
def execution_start():
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
app_descr_json = request.files['file'].read().decode('utf-8')
app_descr = json.loads(app_descr_json)
......@@ -47,7 +47,7 @@ def execution_start():
@catch_exceptions
def execution_restart(execution_id):
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
e = config.api_endpoint.execution_by_id(uid, role, execution_id)
new_id = config.api_endpoint.execution_start(uid, role, e.name, e.description)
......@@ -58,11 +58,11 @@ def execution_restart(execution_id):
@catch_exceptions
def execution_terminate(execution_id):
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
success, message = config.api_endpoint.execution_terminate(uid, role, execution_id)
if not success:
raise zoe_web.exceptions.ZoeException(message)
raise zoe_api.exceptions.ZoeException(message)
return redirect(url_for('web.home_user'))
......@@ -70,19 +70,18 @@ def execution_terminate(execution_id):
@catch_exceptions
def execution_inspect(execution_id):
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
e = config.api_endpoint.execution_by_id(uid, role, execution_id)
services = []
for sid in e['services']:
services.append(cont_api.get(sid))
for s in services:
s['ip'] = list(s['ip_address'].values())[0]
services_info = {}
if e.service_list is not None:
for s in e.service_list:
services_info[s.id] = config.api_endpoint.service_inspect(s)
template_vars = {
"e": e,
"services": services
"services": e.service_list,
"services_info": services_info
}
return render_template('execution_inspect.html', **template_vars)
......@@ -20,9 +20,9 @@ import json
from flask import render_template, request
from zoe_lib.services import ZoeServiceAPI
import zoe_web.config as config
import zoe_web.api_endpoint
from zoe_web.web.utils import get_auth, catch_exceptions
import zoe_api.config as config
import zoe_api.api_endpoint
from zoe_api.web.utils import get_auth, catch_exceptions
guest_id_pattern = re.compile('^\w+$')
......@@ -35,7 +35,7 @@ def index():
@catch_exceptions
def home_user():
uid, role = get_auth(request)
assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
if role == 'user' or role == 'admin':
executions = config.api_endpoint.execution_list(uid, role)
......
{% extends "base_user.html" %}
{% block title %}Inspect execution {{ e['name'] }}{% endblock %}
{% block title %}Inspect execution {{ e.name }}{% endblock %}
{% block content %}
<h2>Detailed information for execution {{ e['name'] }}</h2>
<h2>Detailed information for execution {{ e.name }}</h2>
<div id="contents">
<ul>
<li>Application name: {{ e['application']['name'] }}</li>
<li>Owner: {{ e['owner'] }}</li>
<li>Status: {{ e['status'] }}</li>
<li>Time submitted: <script>format_timestamp("{{ e['time_scheduled'] }}")</script></li>
{% if e['time_started'] == None %}
<li>Application name: {{ e.description['name'] }}</li>
<li>Owner: {{ e.user_id }}</li>
<li>Status: {{ e.status }}</li>
<li>Time submitted: <script>format_timestamp("{{ e.time_submit }}")</script></li>
{% if e.time_start == None %}
<li>Not yet</li>
{% else %}
<li>Time started: <script>format_timestamp("{{ e['time_started'] }}")</script></li>
<li>Time started: <script>format_timestamp("{{ e.time_start }}")</script></li>
{% endif %}
{% if e['time_finished'] == None %}
{% if e.time_end == None %}
<li>Not yet</li>
{% else %}
<li>Time finished: <script>format_timestamp("{{ e['time_finished'] }}")</script></li>
<li>Time finished: <script>format_timestamp("{{ e.time_end }}")</script></li>
{% endif %}
</ul>
{% if e['status'] == 'error' %}
<p>Error message: <code>{{ e['error'] }}</code></p>
{% if e.status == 'error' %}
<p>Error message: <code>{{ e.error_message }}</code></p>
{% endif %}
<div id="container_list">
......
......@@ -17,8 +17,8 @@ import logging
from flask import Response, render_template
from zoe_web.auth.ldap import LDAPAuthenticator
import zoe_web.exceptions
from zoe_api.auth.ldap import LDAPAuthenticator
import zoe_api.exceptions
log = logging.getLogger(__name__)
......@@ -32,11 +32,11 @@ def catch_exceptions(func):
def func_wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except zoe_web.exceptions.ZoeAuthException:
except zoe_api.exceptions.ZoeAuthException:
return missing_auth()
except zoe_web.exceptions.ZoeNotFoundException as e:
except zoe_api.exceptions.ZoeNotFoundException as e:
return error_page(e.message, 404)
except zoe_web.exceptions.ZoeException as e:
except zoe_api.exceptions.ZoeException as e:
return error_page(e, 400)
except Exception as e:
log.exception(str(e))
......@@ -55,12 +55,12 @@ def missing_auth():
def get_auth(request):
auth = request.authorization
if not auth:
raise zoe_web.exceptions.ZoeAuthException
raise zoe_api.exceptions.ZoeAuthException
authenticator = LDAPAuthenticator()
uid, role = authenticator.auth(auth.username, auth.password)
if uid is None:
raise zoe_web.exceptions.ZoeAuthException
raise zoe_api.exceptions.ZoeAuthException
return uid, role
......
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import smtplib
from email.mime.text import MIMEText
import logging
from jinja2 import Template
from zoe_master.state.execution import ExecutionState
from common.configuration import zoe_conf
log = logging.getLogger(__name__)
APP_FINISH_EMAIL_TEMPLATE = """Application {{ name }} has finished executing after {{ runtime }}.
At this URL you can download the execution logs: {{ log_url }}
"""
def generate_log_history_url(execution: ExecutionState) -> str:
zoe_web_log_history_path = '/api/history/logs/'
return 'http://' + zoe_conf().web_server_name + zoe_web_log_history_path + str(execution.id)
def do_duration(seconds):
m, s = divmod(seconds, 60)