Fix inspecting service runtime details (ip address) and centralize log level configuration

zoe_api/entrypoint.py

@@ -41,8 +41,6 @@ def zoe_web_main() -> int:
         logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
     else:
         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
-    logging.getLogger("requests").setLevel(logging.WARNING)
-    logging.getLogger("tornado").setLevel(logging.DEBUG)
 
     log.info("Starting HTTP server...")
     app = Flask(__name__, static_url_path='/does-not-exist')
@@ -51,10 +49,9 @@ def zoe_web_main() -> int:
     zoe_api.db_init.init()
 
     api_endpoint = zoe_api.api_endpoint.APIEndpoint()
-    config.api_endpoint = api_endpoint
 
     app.register_blueprint(zoe_api.rest_api.api_init(api_endpoint))
-    app.register_blueprint(zoe_api.web.web_init())
+    app.register_blueprint(zoe_api.web.web_init(api_endpoint))
 
     http_server = HTTPServer(WSGIContainer(app))
     http_server.listen(args.listen_port, args.listen_address)

zoe_api/rest_api/__init__.py

@@ -35,7 +35,7 @@ def api_init(api_endpoint) -> Blueprint:
     api.add_resource(ExecutionAPI, API_PATH + '/execution/<int:execution_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
     api.add_resource(ExecutionDeleteAPI, API_PATH + '/execution/delete/<int:execution_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
     api.add_resource(ExecutionCollectionAPI, API_PATH + '/execution', resource_class_kwargs={'api_endpoint': api_endpoint})
-    api.add_resource(ServiceAPI, API_PATH + '/service/<int:container_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
+    api.add_resource(ServiceAPI, API_PATH + '/service/<int:service_id>', resource_class_kwargs={'api_endpoint': api_endpoint})
     api.add_resource(QueryAPI, API_PATH + '/query', resource_class_kwargs={'api_endpoint': api_endpoint})
 
     return api_bp

zoe_api/web/__init__.py

@@ -13,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from flask import Blueprint
+from flask import Blueprint, g
 import zoe_api.web.start
 import zoe_api.web.executions
 
 from zoe_lib.version import ZOE_API_VERSION, ZOE_VERSION
 
 
-def web_init() -> Blueprint:
+def web_init(api_endpoint) -> Blueprint:
+    def before_request():
+        g.api_endpoint = api_endpoint
+
     web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')
 
     web_bp.context_processor(inject_version)
@@ -34,9 +37,13 @@ def web_init() -> Blueprint:
     web_bp.add_url_rule('/executions/terminate/<int:execution_id>', 'execution_terminate', zoe_api.web.executions.execution_terminate)
     web_bp.add_url_rule('/executions/inspect/<int:execution_id>', 'execution_inspect', zoe_api.web.executions.execution_inspect)
 
+    web_bp.before_request(before_request)
+
     return web_bp
 
 
+
+
 def inject_version():
     return {
         'zoe_version': ZOE_VERSION,

zoe_api/web/executions.py

@@ -15,10 +15,9 @@
 
 import json
 
-from flask import render_template, request, redirect, url_for
+from flask import render_template, request, redirect, url_for, g
 
 from zoe_api.web.utils import get_auth, catch_exceptions
-import zoe_api.config as config
 import zoe_api.api_endpoint
 import zoe_api.exceptions
 
@@ -33,13 +32,14 @@ def execution_define():
 @catch_exceptions
 def execution_start():
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     app_descr_json = request.files['file'].read().decode('utf-8')
     app_descr = json.loads(app_descr_json)
     exec_name = request.form['exec_name']
 
-    new_id = config.api_endpoint.execution_start(uid, role, exec_name, app_descr)
+    new_id = api_endpoint.execution_start(uid, role, exec_name, app_descr)
 
     return redirect(url_for('web.execution_inspect', execution_id=new_id))
 
@@ -47,10 +47,11 @@ def execution_start():
 @catch_exceptions
 def execution_restart(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    e = config.api_endpoint.execution_by_id(uid, role, execution_id)
-    new_id = config.api_endpoint.execution_start(uid, role, e.name, e.description)
+    e = api_endpoint.execution_by_id(uid, role, execution_id)
+    new_id = api_endpoint.execution_start(uid, role, e.name, e.description)
 
     return redirect(url_for('web.execution_inspect', execution_id=new_id))
 
@@ -58,9 +59,10 @@ def execution_restart(execution_id):
 @catch_exceptions
 def execution_terminate(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    success, message = config.api_endpoint.execution_terminate(uid, role, execution_id)
+    success, message = api_endpoint.execution_terminate(uid, role, execution_id)
     if not success:
         raise zoe_api.exceptions.ZoeException(message)
 
@@ -70,9 +72,10 @@ def execution_terminate(execution_id):
 @catch_exceptions
 def execution_delete(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    success, message = config.api_endpoint.execution_delete(uid, role, execution_id)
+    success, message = api_endpoint.execution_delete(uid, role, execution_id)
     if not success:
         raise zoe_api.exceptions.ZoeException(message)
 
@@ -82,18 +85,17 @@ def execution_delete(execution_id):
 @catch_exceptions
 def execution_inspect(execution_id):
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    e = config.api_endpoint.execution_by_id(uid, role, execution_id)
+    e = api_endpoint.execution_by_id(uid, role, execution_id)
 
-    services_info = {}
-    if e.service_list is not None:
-        for s in e.service_list:
-            services_info[s.id] = config.api_endpoint.service_inspect(uid, role, s)
+    services_info = []
+    for s in e.services:
+        services_info.append(api_endpoint.service_by_id(uid, role, s.id))
 
     template_vars = {
         "e": e,
-        "services": e.service_list,
         "services_info": services_info
     }
     return render_template('execution_inspect.html', **template_vars)

zoe_api/web/start.py

@@ -33,10 +33,11 @@ def index():
 @catch_exceptions
 def home_user():
     uid, role = get_auth(request)
-    assert isinstance(config.api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    api_endpoint = g.api_endpoint
+    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
     if role == 'user' or role == 'admin':
-        executions = config.api_endpoint.execution_list(uid, role)
+        executions = api_endpoint.execution_list(uid, role)
 
         template_vars = {
             'executions': executions,
@@ -51,9 +52,9 @@ def home_user():
         }
 
         app_descr = json.load(open('contrib/zoeapps/eurecom_aml_lab.json', 'r'))
-        execution = config.api_endpoint.execution_list(uid, role, name='aml-lab')
+        execution = api_endpoint.execution_list(uid, role, name='aml-lab')
         if len(execution) == 0 or execution[0]['status'] == 'terminated' or execution[0]['status'] == 'finished':
-            config.api_endpoint.execution_start(uid, role, 'aml-lab', app_descr)
+            api_endpoint.execution_start(uid, role, 'aml-lab', app_descr)
             template_vars['execution_status'] = 'submitted'
             return render_template('home_guest.html', **template_vars)
         else:
@@ -63,7 +64,6 @@ def home_user():
                 return render_template('home_guest.html', **template_vars)
             else:
                 template_vars['refresh'] = -1
-                cont_api = ZoeServiceAPI(get_conf().master_url, guest_identifier, guest_password)
                 template_vars['execution_status'] = execution['status']
                 for c_id in execution['services']:
                     c = cont_api.get(c_id)

zoe_api/web/templates/execution_inspect.html

@@ -9,12 +9,12 @@
     <li>Status: {{ e.status }}</li>
     <li>Time submitted: <script>format_timestamp("{{ e.time_submit }}")</script></li>
     {% if e.time_start == None %}
-    <li>Not yet</li>
+    <li>Time started: not yet</li>
     {% else %}
     <li>Time started: <script>format_timestamp("{{ e.time_start }}")</script></li>
     {% endif %}
     {% if e.time_end == None %}
-    <li>Not yet</li>
+    <li>Time finished: not yet</li>
     {% else %}
     <li>Time finished: <script>format_timestamp("{{ e.time_end }}")</script></li>
     {%  endif %}
@@ -28,11 +28,11 @@
 <p>Services:</p>
 {% endif %}
 <ul>
-{% for s in services %}
+{% for s in services_info %}
     <li class="container_name" id="{{ s['id'] }}">{{ s['name'] }}</li>
     <ul>
-    {% for p in s['ports'] %}
-        <li><a href="{{ p['protocol'] }}://{{ s['ip'] }}:{{ p['port_number'] }}{{ p['path'] }}">{{ p['name'] }}</a></li>
+    {% for p in s['description']['ports'] %}
+        <li><a href="{{ p['protocol'] }}://{{ s['ip_address'] }}:{{ p['port_number'] }}{{ p['path'] }}">{{ p['name'] }}</a></li>
     {% endfor %}
     </ul>
 {% endfor %}

zoe_cmd/entrypoint.py

@@ -93,9 +93,9 @@ def exec_get_cmd(args):
         print('Application name: {}'.format(app['name']))
         for c_id in execution['services']:
             c = cont_api.get(c_id)
-            ip = list(c['ip_address'].values())[0]  # FIXME how to decide which network is the right one?
-            print('Service {} (ID: {})'.format(c['name'], c['id']))
-            for p in c['ports']:
+            ip = c['ip_address']
+            print('Service {} (ID: {}, {})'.format(c['name'], c['id'], c['status']))
+            for p in c['description']['ports']:
                 print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'], ip, p['port_number'], p['path']))
 
 

zoe_lib/sql_manager.py

@@ -216,7 +216,8 @@ class Execution(Base):
             'time_start': None if self.time_start is None else self.time_start.timestamp(),
             'time_end': None if self.time_end is None else self.time_end.timestamp(),
             'status': self._status,
-            'error_message': self.error_message
+            'error_message': self.error_message,
+            'services': [s.id for s in self.services]
         }
 
     def __eq__(self, other):
@@ -294,13 +295,15 @@ class Service(Base):
 
     def serialize(self):
         return {
+            'id': self.id,
             'name': self.name,
             'status': self.status,
             'error_message': self.error_message,
             'execution_id': self.execution_id,
             'description': self.description,
             'service_group': self.service_group,
-            'docker_id': self.docker_id
+            'docker_id': self.docker_id,
+            'ip_address': self.ip_address
         }
 
     def __eq__(self, other):
@@ -321,3 +324,11 @@ class Service(Base):
 
     def set_active(self, docker_id):
         self.sql_manager.service_update(self.id, status=self.ACTIVE_STATUS, docker_id=docker_id)
+
+    @property
+    def ip_address(self):
+        if self.status != self.ACTIVE_STATUS:
+            return {}
+        swarm = SwarmClient(get_conf())
+        s_info = swarm.inspect_container(self.docker_id)
+        return s_info['ip_address'][get_conf().overlay_network_name]

zoe_master/entrypoint.py

@@ -45,12 +45,6 @@ def main():
     else:
         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
 
-    logging.getLogger('kazoo').setLevel(logging.WARNING)
-    logging.getLogger('requests').setLevel(logging.WARNING)
-    logging.getLogger('urllib3').setLevel(logging.WARNING)
-    logging.getLogger('docker').setLevel(logging.INFO)
-    logging.getLogger("tornado").setLevel(logging.DEBUG)
-
     log.info("Initializing DB manager")
     config.singletons['sql_manager'] = SQLManager(args)