Rename zoe-web in zoe-api

zoe-web.py → zoe-api.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from zoe_web.entrypoint import zoe_web_main
+from zoe_api.entrypoint import zoe_web_main
 
 if __name__ == "__main__":
     zoe_web_main()

zoe_web/api_endpoint.py → zoe_api/api_endpoint.py

@@ -16,31 +16,31 @@
 import logging
 import re
 
-from zoe_web.config import get_conf
-import zoe_web.master_api
+from zoe_api.config import get_conf
+import zoe_api.master_api
 import zoe_lib.sql_manager
 import zoe_lib.applications
 import zoe_lib.exceptions
-import zoe_web.exceptions
+import zoe_api.exceptions
 
 log = logging.getLogger(__name__)
 
 
 class APIEndpoint:
     """
-    :type master: zoe_web.master_api.APIManager
+    :type master: zoe_api.master_api.APIManager
     :type sql: zoe_lib.sql_manager.SQLManager
     """
     def __init__(self):
-        self.master = zoe_web.master_api.APIManager()
+        self.master = zoe_api.master_api.APIManager()
         self.sql = zoe_lib.sql_manager.SQLManager(get_conf())
 
     def execution_by_id(self, uid, role, execution_id):
         e = self.sql.execution_list(id=execution_id, only_one=True)
         if e is None:
-            raise zoe_web.exceptions.ZoeNotFoundException('No such execution')
+            raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
         if e.user_id != uid and role != 'admin':
-            raise zoe_web.exceptions.ZoeAuthException()
+            raise zoe_api.exceptions.ZoeAuthException()
         return e
 
     def execution_list(self, uid, role, **filters):
@@ -52,28 +52,40 @@ class APIEndpoint:
         try:
             zoe_lib.applications.app_validate(application_description)
         except zoe_lib.exceptions.InvalidApplicationDescription as e:
-            raise zoe_web.exceptions.ZoeException('Invalid application description: ' + e.message)
+            raise zoe_api.exceptions.ZoeException('Invalid application description: ' + e.message)
 
         if 3 > len(exec_name) > 128:
-            raise zoe_web.exceptions.ZoeException("Execution name must be between 4 and 128 characters long")
+            raise zoe_api.exceptions.ZoeException("Execution name must be between 4 and 128 characters long")
         if not re.match(r'^[a-zA-Z0-9\-]+$', exec_name):
-            raise zoe_web.exceptions.ZoeException("Execution name can contain only letters, numbers and dashes. '{}' is not valid.".format(exec_name))
+            raise zoe_api.exceptions.ZoeException("Execution name can contain only letters, numbers and dashes. '{}' is not valid.".format(exec_name))
 
         new_id = self.sql.execution_new(exec_name, uid, application_description)
         success, message = self.master.execution_start(new_id)
         if not success:
-            raise zoe_web.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up.')
+            raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up.')
         return new_id
 
     def execution_terminate(self, uid, role, exec_id):
         e = self.sql.execution_list(id=exec_id, only_one=True)
         if e is None:
-            raise zoe_web.exceptions.ZoeNotFoundException('No such execution')
+            raise zoe_api.exceptions.ZoeNotFoundException('No such execution')
 
         if e.user_id != uid and role != 'admin':
-            raise zoe_web.exceptions.ZoeAuthException()
+            raise zoe_api.exceptions.ZoeAuthException()
 
         if e.is_active():
             return self.master.execution_terminate(exec_id)
         else:
-            raise zoe_web.exceptions.ZoeException('Execution is not running')
+            raise zoe_api.exceptions.ZoeException('Execution is not running')
+
+    def service_inspect(self, service):
+        return self.master.service_inspect(service.id)
+
+    def retry_submit_error_executions(self):
+        waiting_execs = self.sql.execution_list(status=zoe_lib.sql_manager.Execution.SUBMIT_STATUS)
+        if waiting_execs is None or len(waiting_execs) == 0:
+            return
+        e = waiting_execs.pop(0)
+        success, message = self.master.execution_start(e.id)
+        if not success:
+            log.warning('Zoe Master unavailable, execution {} still waiting'.format(e.id))

zoe_web/auth/ldap.py → zoe_api/auth/ldap.py

@@ -16,13 +16,13 @@ import logging
 
 import ldap
 
-import zoe_web.auth.base
-from zoe_web.config import get_conf
+import zoe_api.auth.base
+from zoe_api.config import get_conf
 
 log = logging.getLogger(__name__)
 
 
-class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
+class LDAPAuthenticator(zoe_api.auth.base.BaseAuthenticator):
     def __init__(self):
         self.connection = ldap.initialize(get_conf().ldap_server_uri)
         self.base_dn = get_conf().ldap_base_dn
@@ -31,13 +31,13 @@ class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
 
     def auth(self, username, password):
         search_filter = "uid=" + username
-        uid_number = None
+        uid = None
         role = 'guest'
         try:
             self.connection.bind_s(self.bind_user, self.bind_password)
             result = self.connection.search_s(self.base_dn, ldap.SCOPE_SUBTREE, search_filter)
             user_dict = result[0][1]
-            uid_number = int(user_dict['uidNumber'][0])
+            uid = username
             gid_numbers = [int(x) for x in user_dict['gidNumber']]
             if get_conf().ldap_admin_gid in gid_numbers:
                 role = 'admin'
@@ -52,4 +52,4 @@ class LDAPAuthenticator(zoe_web.auth.base.BaseAuthenticator):
             log.exception("LDAP exception")
         finally:
             self.connection.unbind_s()
-        return uid_number, role
+        return uid, role

zoe_web/config.py → zoe_api/config.py

@@ -18,8 +18,8 @@ from zoe_lib.configargparse import ArgumentParser, Namespace
 api_endpoint = None  # singleton
 
 config_paths = [
-    'zoe-web.conf',
-    '/etc/zoe/zoe-web.conf'
+    'zoe-api.conf',
+    '/etc/zoe/zoe-api.conf'
 ]
 
 _conf = None

zoe_web/db_init.py → zoe_api/db_init.py

@@ -17,7 +17,7 @@ import psycopg2
 import psycopg2.extras
 
 import zoe_lib.exceptions
-from zoe_web.config import get_conf
+from zoe_api.config import get_conf
 
 SQL_SCHEMA_VERSION = 0  # ---> Increment this value every time the schema changes !!! <---
 
@@ -50,7 +50,7 @@ def create_tables(cur):
     cur.execute('''CREATE TABLE execution (
         id SERIAL PRIMARY KEY,
         name TEXT NOT NULL,
-        user_id INTEGER,
+        user_id TEXT NOT NULL,
         description JSON NOT NULL,
         status TEXT NOT NULL,
         execution_manager_id TEXT NULL,
@@ -61,10 +61,13 @@ def create_tables(cur):
         )''')
     cur.execute('''CREATE TABLE service (
         id SERIAL PRIMARY KEY,
+        execution_id INT REFERENCES execution,
+        name TEXT NOT NULL ,
         status TEXT NOT NULL,
-        stage TEXT,
-        error TEXT NULL,
-        execution_manager_id TEXT
+        error_message TEXT NULL DEFAULT NULL,
+        description JSON NOT NULL,
+        service_group TEXT NOT NULL,
+        docker_id TEXT NULL DEFAULT NULL
         )''')
 
 

zoe_web/entrypoint.py → zoe_api/entrypoint.py

@@ -18,15 +18,15 @@ import logging
 from flask import Flask
 from tornado.wsgi import WSGIContainer
 from tornado.httpserver import HTTPServer
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
-import zoe_web.config as config
-import zoe_web.db_init
-import zoe_web.api_endpoint
-import zoe_web.rest_api
-import zoe_web.web
+import zoe_api.config as config
+import zoe_api.db_init
+import zoe_api.api_endpoint
+import zoe_api.rest_api
+import zoe_api.web
 
-log = logging.getLogger("zoe_web")
+log = logging.getLogger("zoe_api")
 LOG_FORMAT = '%(asctime)-15s %(levelname)s %(name)s (%(threadName)s): %(message)s'
 
 
@@ -48,16 +48,20 @@ def zoe_web_main() -> int:
     app = Flask(__name__, static_url_path='/does-not-exist')
     app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
 
-    app.register_blueprint(zoe_web.rest_api.api_init())
-    app.register_blueprint(zoe_web.web.web_init())
+    app.register_blueprint(zoe_api.rest_api.api_init())
+    app.register_blueprint(zoe_api.web.web_init())
 
-    zoe_web.db_init.init()
+    zoe_api.db_init.init()
 
-    config.api_endpoint = zoe_web.api_endpoint.APIEndpoint()
+    config.api_endpoint = zoe_api.api_endpoint.APIEndpoint()
 
     http_server = HTTPServer(WSGIContainer(app))
     http_server.listen(args.listen_port, args.listen_address)
     ioloop = IOLoop.instance()
+
+    retry_cb = PeriodicCallback(config.api_endpoint.retry_submit_error_executions, 30000)
+    retry_cb.start()
+
     try:
         ioloop.start()
     except KeyboardInterrupt:

zoe_web/master_api.py → zoe_api/master_api.py

@@ -17,14 +17,14 @@ import logging
 
 import zmq
 
-import zoe_web.config as config
+import zoe_api.config as config
 
 log = logging.getLogger(__name__)
 
 
 class APIManager:
     REQUEST_TIMEOUT = 2500
-    REQUEST_RETRIES = 3
+    REQUEST_RETRIES = 1
 
     def __init__(self):
         self.context = zmq.Context(1)
@@ -84,3 +84,10 @@ class APIManager:
             'exec_id': exec_id
         }
         return self._request_reply(msg)
+
+    def service_inspect(self, service_id):
+        msg = {
+            'command': 'service_inspect',
+            'service_id': service_id
+        }
+        return self._request_reply(msg)

zoe_web/rest_api/__init__.py → zoe_api/rest_api/__init__.py

@@ -17,12 +17,12 @@ import sys
 
 from flask import Blueprint
 from flask_restful import Api
-from zoe_web.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
-from zoe_web.rest_api.info import InfoAPI
-from zoe_web.rest_api.service import ServiceAPI
+from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
+from zoe_api.rest_api.info import InfoAPI
+from zoe_api.rest_api.service import ServiceAPI
 
 from zoe_lib.version import ZOE_API_VERSION
-from zoe_web.rest_api.query import QueryAPI
+from zoe_api.rest_api.query import QueryAPI
 
 API_PATH = '/api/' + ZOE_API_VERSION
 

zoe_web/rest_api/execution.py → zoe_api/rest_api/execution.py

@@ -22,10 +22,10 @@ import zoe_lib.exceptions
 import zoe_lib.applications
 import zoe_lib.sql_manager
 
-from zoe_web.rest_api.utils import catch_exceptions, get_auth
-import zoe_web.exceptions
-import zoe_web.config as config
-import zoe_web.api_endpoint
+from zoe_api.rest_api.utils import catch_exceptions, get_auth
+import zoe_api.exceptions
+import zoe_api.config as config
+import zoe_api.api_endpoint
 
 
 class ExecutionAPI(Resource):
@@ -33,7 +33,7 @@ class ExecutionAPI(Resource):
     def get(self, execution_id):
         uid, role = get_auth(request)
 
-        assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+        assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
         e = config.api_endpoint.execution_by_id(uid, role, execution_id)
 
         return e.serialize()
@@ -48,10 +48,10 @@ class ExecutionAPI(Resource):
         """
         uid, role = get_auth(request)
 
-        assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+        assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
         success, message = config.api_endpoint.execution_terminate(uid, role, execution_id)
         if not success:
-            raise zoe_web.exceptions.ZoeRestAPIException(message, 400)
+            raise zoe_api.exceptions.ZoeRestAPIException(message, 400)
 
         return '', 204
 
@@ -66,7 +66,7 @@ class ExecutionCollectionAPI(Resource):
         """
         uid, role = get_auth(request)
 
-        assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+        assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
         execs = config.api_endpoint.execution_list(uid, role)
         return [e.serialize() for e in execs]
 
@@ -78,12 +78,12 @@ class ExecutionCollectionAPI(Resource):
         """
         uid, role = get_auth(request)
 
-        assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+        assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
         try:
             data = request.get_json()
         except BadRequest:
-            raise zoe_web.exceptions.ZoeRestAPIException('Error decoding JSON data')
+            raise zoe_api.exceptions.ZoeRestAPIException('Error decoding JSON data')
 
         application_description = data['application']
         exec_name = data['name']

zoe_web/rest_api/info.py → zoe_api/rest_api/info.py

@@ -17,7 +17,7 @@ from flask_restful import Resource
 
 from zoe_lib.version import ZOE_API_VERSION, ZOE_APPLICATION_FORMAT_VERSION, ZOE_VERSION
 from zoe_master.config import get_conf
-from zoe_web.rest_api.utils import catch_exceptions
+from zoe_api.rest_api.utils import catch_exceptions
 
 
 class InfoAPI(Resource):

zoe_web/rest_api/query.py → zoe_api/rest_api/query.py

@@ -16,17 +16,17 @@
 from flask_restful import Resource, request
 from werkzeug.exceptions import BadRequest
 
-from zoe_web.exceptions import ZoeRestAPIException
-import zoe_web.config as config
-import zoe_web.api_endpoint
-from zoe_web.rest_api.utils import catch_exceptions, get_auth
+from zoe_api.exceptions import ZoeRestAPIException
+import zoe_api.config as config
+import zoe_api.api_endpoint
+from zoe_api.rest_api.utils import catch_exceptions, get_auth
 
 
 class QueryAPI(Resource):
     @catch_exceptions
     def post(self):
         uid, role = get_auth(request)
-        assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+        assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
         try:
             data = request.get_json()

zoe_web/rest_api/service.py → zoe_api/rest_api/service.py

@@ -18,9 +18,8 @@ import time
 
 from flask_restful import Resource, request
 
-from zoe_lib.exceptions import ZoeRestAPIException
-from zoe_web.config import singletons
-from zoe_web.rest_api.utils import catch_exceptions
+from zoe_api.exceptions import ZoeRestAPIException
+from zoe_api.rest_api.utils import catch_exceptions
 
 log = logging.getLogger(__name__)
 

zoe_web/rest_api/utils.py → zoe_api/rest_api/utils.py

@@ -15,8 +15,8 @@
 
 import logging
 
-from zoe_web.exceptions import ZoeRestAPIException, ZoeNotFoundException, ZoeAuthException, ZoeException
-from zoe_web.auth.ldap import LDAPAuthenticator
+from zoe_api.exceptions import ZoeRestAPIException, ZoeNotFoundException, ZoeAuthException, ZoeException
+from zoe_api.auth.ldap import LDAPAuthenticator
 
 log = logging.getLogger(__name__)
 

zoe_web/web/__init__.py → zoe_api/web/__init__.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 from flask import Blueprint
-import zoe_web.web.start
-import zoe_web.web.executions
+import zoe_api.web.start
+import zoe_api.web.executions
 
 from zoe_lib.version import ZOE_API_VERSION, ZOE_VERSION
 
@@ -25,14 +25,14 @@ def web_init() -> Blueprint:
 
     web_bp.context_processor(inject_version)
 
-    web_bp.add_url_rule('/', 'index', zoe_web.web.start.index)
-    web_bp.add_url_rule('/user', 'home_user', zoe_web.web.start.home_user)
+    web_bp.add_url_rule('/', 'index', zoe_api.web.start.index)
+    web_bp.add_url_rule('/user', 'home_user', zoe_api.web.start.home_user)
 
-    web_bp.add_url_rule('/executions/new', 'execution_define', zoe_web.web.executions.execution_define)
-    web_bp.add_url_rule('/executions/start', 'execution_start', zoe_web.web.executions.execution_start, methods=['POST'])
-    web_bp.add_url_rule('/executions/restart/<int:execution_id>', 'execution_restart', zoe_web.web.executions.execution_restart)
-    web_bp.add_url_rule('/executions/terminate/<int:execution_id>', 'execution_terminate', zoe_web.web.executions.execution_terminate)
-    web_bp.add_url_rule('/executions/inspect/<int:execution_id>', 'execution_inspect', zoe_web.web.executions.execution_inspect)
+    web_bp.add_url_rule('/executions/new', 'execution_define', zoe_api.web.executions.execution_define)
+    web_bp.add_url_rule('/executions/start', 'execution_start', zoe_api.web.executions.execution_start, methods=['POST'])
+    web_bp.add_url_rule('/executions/restart/<int:execution_id>', 'execution_restart', zoe_api.web.executions.execution_restart)
+    web_bp.add_url_rule('/executions/terminate/<int:execution_id>', 'execution_terminate', zoe_api.web.executions.execution_terminate)
+    web_bp.add_url_rule('/executions/inspect/<int:execution_id>', 'execution_inspect', zoe_api.web.executions.execution_inspect)
 
     return web_bp
 

zoe_web/web/executions.py → zoe_api/web/executions.py

@@ -17,10 +17,10 @@ import json
 
 from flask import render_template, request, redirect, url_for
 
-from zoe_web.web.utils import get_auth, catch_exceptions
-import zoe_web.config as config
-import zoe_web.api_endpoint
-import zoe_web.exceptions
+from zoe_api.web.utils import get_auth, catch_exceptions
+import zoe_api.config as config
+import zoe_api.api_endpoint
+import zoe_api.exceptions
 
 
 @catch_exceptions
@@ -33,7 +33,7 @@ def execution_define():
 @catch_exceptions
 def execution_start():
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     app_descr_json = request.files['file'].read().decode('utf-8')
     app_descr = json.loads(app_descr_json)
@@ -47,7 +47,7 @@ def execution_start():
 @catch_exceptions
 def execution_restart(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     e = config.api_endpoint.execution_by_id(uid, role, execution_id)
     new_id = config.api_endpoint.execution_start(uid, role, e.name, e.description)
@@ -58,11 +58,11 @@ def execution_restart(execution_id):
 @catch_exceptions
 def execution_terminate(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     success, message = config.api_endpoint.execution_terminate(uid, role, execution_id)
     if not success:
-        raise zoe_web.exceptions.ZoeException(message)
+        raise zoe_api.exceptions.ZoeException(message)
 
     return redirect(url_for('web.home_user'))
 
@@ -70,19 +70,18 @@ def execution_terminate(execution_id):
 @catch_exceptions
 def execution_inspect(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     e = config.api_endpoint.execution_by_id(uid, role, execution_id)
 
-    services = []
-    for sid in e['services']:
-        services.append(cont_api.get(sid))
-
-    for s in services:
-        s['ip'] = list(s['ip_address'].values())[0]
+    services_info = {}
+    if e.service_list is not None:
+        for s in e.service_list:
+            services_info[s.id] = config.api_endpoint.service_inspect(s)
 
     template_vars = {
         "e": e,
-        "services": services
+        "services": e.service_list,
+        "services_info": services_info
     }
     return render_template('execution_inspect.html', **template_vars)

zoe_web/web/start.py → zoe_api/web/start.py

@@ -20,9 +20,9 @@ import json
 from flask import render_template, request
 from zoe_lib.services import ZoeServiceAPI
 
-import zoe_web.config as config
-import zoe_web.api_endpoint
-from zoe_web.web.utils import get_auth, catch_exceptions
+import zoe_api.config as config
+import zoe_api.api_endpoint
+from zoe_api.web.utils import get_auth, catch_exceptions
 
 guest_id_pattern = re.compile('^\w+$')
 
@@ -35,7 +35,7 @@ def index():
 @catch_exceptions
 def home_user():
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_web.api_endpoint.APIEndpoint)
+    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     if role == 'user' or role == 'admin':
         executions = config.api_endpoint.execution_list(uid, role)