Convert the web interface to pure tornado+jinja2 templates

.pylintrc

@@ -47,7 +47,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=line-too-long,logging-format-interpolation,too-few-public-methods,too-many-instance-attributes,fixme,too-many-branches,file-ignored,global-statement,redefined-variable-type,no-self-use,too-many-statements
+disable=line-too-long,logging-format-interpolation,too-few-public-methods,too-many-instance-attributes,fixme,too-many-branches,file-ignored,global-statement,redefined-variable-type,no-self-use,too-many-statements,locally-disabled,arguments-differ
 
 
 [REPORTS]
@@ -309,7 +309,7 @@ int-import-graph=
 [CLASSES]
 
 # List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__
+defining-attr-methods=__init__,initialize
 
 # List of valid names for the first argument in a class method.
 valid-classmethod-first-arg=cls

requirements.txt

-Flask>=0.10.1
-Flask-RESTful
+Jinja2>=2.8
 requests>=2.9.1
 docker-py==1.7.2
 tornado>=4.3

requirements_tests.txt

-pylint
+pylint>=1.6.4
 vulture
 doc8
 sphinx

zoe_api/entrypoint.py

@@ -28,6 +28,7 @@ import zoe_api.api_endpoint
 import zoe_api.rest_api
 import zoe_api.web
 import zoe_api.auth.ldap
+from zoe_api.web.custom_request_handler import JinjaApp
 
 log = logging.getLogger("zoe_api")
 LOG_FORMAT = '%(asctime)-15s %(levelname)s %(name)s (%(threadName)s): %(message)s'
@@ -55,9 +56,11 @@ def zoe_web_main() -> int:
 
     app_settings = {
         'static_path': os.path.join(os.path.dirname(__file__), "web", "static"),
+        'template_path': os.path.join(os.path.dirname(__file__), "web", "templates"),
         'debug': args.debug
     }
     app = Application(zoe_api.web.web_init(api_endpoint) + zoe_api.rest_api.api_init(api_endpoint), **app_settings)
+    JinjaApp.init_app(app)
 
     log.info("Starting HTTP server...")
     http_server = HTTPServer(app)

zoe_api/rest_api/discovery.py

@@ -17,7 +17,7 @@
 
 from tornado.web import RequestHandler
 
-from zoe_api.api_endpoint import APIEndpoint
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
 from zoe_api.rest_api.utils import catch_exceptions
 
 
@@ -43,3 +43,7 @@ class DiscoveryAPI(RequestHandler):
         }
 
         self.write(ret)
+
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass

zoe_api/rest_api/execution.py

@@ -20,7 +20,7 @@ import tornado.escape
 
 from zoe_api.rest_api.utils import catch_exceptions, get_auth
 import zoe_api.exceptions
-from zoe_api.api_endpoint import APIEndpoint
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
 
 
 class ExecutionAPI(RequestHandler):
@@ -55,6 +55,10 @@ class ExecutionAPI(RequestHandler):
 
         self.set_status(204)
 
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass
+
 
 class ExecutionDeleteAPI(RequestHandler):
     """The ExecutionDelete API endpoints."""
@@ -79,6 +83,10 @@ class ExecutionDeleteAPI(RequestHandler):
 
         self.set_status(204)
 
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass
+
 
 class ExecutionCollectionAPI(RequestHandler):
     """The Execution Collection API endpoints."""
@@ -119,3 +127,7 @@ class ExecutionCollectionAPI(RequestHandler):
 
         self.set_status(201)
         self.write({'execution_id': new_id})
+
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass

zoe_api/rest_api/info.py

@@ -18,6 +18,8 @@
 from tornado.web import RequestHandler
 
 from zoe_api.rest_api.utils import catch_exceptions
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
+
 from zoe_lib.config import get_conf
 from zoe_lib.version import ZOE_API_VERSION, ZOE_APPLICATION_FORMAT_VERSION, ZOE_VERSION
 
@@ -40,3 +42,7 @@ class InfoAPI(RequestHandler):
         }
 
         self.write(ret)
+
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass

zoe_api/rest_api/service.py

@@ -21,7 +21,7 @@ from tornado.web import RequestHandler
 import tornado.gen
 
 from zoe_api.rest_api.utils import catch_exceptions, get_auth
-from zoe_api.api_endpoint import APIEndpoint
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
 
 log = logging.getLogger(__name__)
 
@@ -42,6 +42,10 @@ class ServiceAPI(RequestHandler):
 
         self.write(service.serialize())
 
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass
+
 
 class ServiceLogsAPI(RequestHandler):
     """The Service logs API endpoint."""
@@ -57,10 +61,11 @@ class ServiceLogsAPI(RequestHandler):
 
     @catch_exceptions
     @tornado.gen.engine
-    def get(self, service_id) -> dict:
+    def get(self, service_id):
         """HTTP GET method."""
 
-        def cb(callback):
+        def new_log_line_cb(callback):
+            """Task callback"""
             if self.connection_closed:
                 tmp_line = None
             else:
@@ -72,7 +77,7 @@ class ServiceLogsAPI(RequestHandler):
         log_gen = self.api_endpoint.service_logs(uid, role, service_id, stream=True)
 
         while True:
-            log_line = yield tornado.gen.Task(cb)
+            log_line = yield tornado.gen.Task(new_log_line_cb)
             if log_line is None:
                 break
             else:
@@ -80,3 +85,7 @@ class ServiceLogsAPI(RequestHandler):
                 yield self.flush()
 
         self.finish()
+
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass

zoe_api/rest_api/utils.py

@@ -18,8 +18,15 @@
 import base64
 import logging
 
+import tornado.web
+
+from zoe_lib.config import get_conf
+
 from zoe_api.exceptions import ZoeRestAPIException, ZoeNotFoundException, ZoeAuthException, ZoeException
 from zoe_api.auth.ldap import LDAPAuthenticator
+from zoe_api.auth.file import PlainTextAuthenticator
+from zoe_api.auth.base import BaseAuthenticator
+
 
 log = logging.getLogger(__name__)
 
@@ -59,16 +66,21 @@ def catch_exceptions(func):
     return func_wrapper
 
 
-def get_auth(request):
+def get_auth(handler: tornado.web.RequestHandler):
     """Try to authenticate a request."""
-    auth_header = request.request.headers.get('Authorization')
+    auth_header = handler.request.headers.get('Authorization')
     if auth_header is None or not auth_header.startswith('Basic '):
         raise ZoeRestAPIException('missing or wrong authentication information', 401, {'WWW-Authenticate': 'Basic realm="Login Required"'})
 
     auth_decoded = base64.decodebytes(bytes(auth_header[6:], 'ascii')).decode('utf-8')
     username, password = auth_decoded.split(':', 2)
 
-    authenticator = LDAPAuthenticator()
+    if get_conf().auth_type == 'text':
+        authenticator = PlainTextAuthenticator()  # type: BaseAuthenticator
+    elif get_conf().auth_type == 'ldap':
+        authenticator = LDAPAuthenticator()
+    else:
+        raise ZoeException('Configuration error, unknown authentication method: {}'.format(get_conf().auth_type))
     uid, role = authenticator.auth(username, password)
     if uid is None:
         raise ZoeRestAPIException('missing or wrong authentication information', 401, {'WWW-Authenticate': 'Basic realm="Login Required"'})

zoe_api/web/__init__.py

@@ -36,15 +36,15 @@ def web_init(api_endpoint) -> List[tornado.web.URLSpec]:
         'api_endpoint': api_endpoint
     }
     web_routes = [
-        tornado.web.url(r'/', zoe_api.web.start.index, route_args, name='root'),
-        tornado.web.url(r'/user', zoe_api.web.start.home_user, route_args, name='home_user'),
-
-        tornado.web.url(r'/executions/new', zoe_api.web.executions.execution_define, route_args, name='execution_new'),
-        tornado.web.url(r'/executions/start', zoe_api.web.executions.execution_start, route_args, name='execution_start'),
-        tornado.web.url(r'/executions/restart/([0-9]+)', zoe_api.web.executions.execution_restart, route_args, name='execution_restart'),
-        tornado.web.url(r'/executions/terminate/([0-9]+)', zoe_api.web.executions.execution_terminate, route_args, name='execution_terminate'),
-        tornado.web.url(r'/executions/delete/([0-9]+)', zoe_api.web.executions.execution_delete, route_args, name='execution_delete'),
-        tornado.web.url(r'/executions/inspect/([0-9]+)', zoe_api.web.executions.execution_inspect, route_args, name='execution_inspect')
+        tornado.web.url(r'/', zoe_api.web.start.RootWeb, route_args, name='root'),
+        tornado.web.url(r'/user', zoe_api.web.start.HomeWeb, route_args, name='home_user'),
+
+        tornado.web.url(r'/executions/new', zoe_api.web.executions.ExecutionDefineWeb, route_args, name='execution_define'),
+        tornado.web.url(r'/executions/start', zoe_api.web.executions.ExecutionStartWeb, route_args, name='execution_start'),
+        tornado.web.url(r'/executions/restart/([0-9]+)', zoe_api.web.executions.ExecutionRestartWeb, route_args, name='execution_restart'),
+        tornado.web.url(r'/executions/terminate/([0-9]+)', zoe_api.web.executions.ExecutionTerminateWeb, route_args, name='execution_terminate'),
+        tornado.web.url(r'/executions/delete/([0-9]+)', zoe_api.web.executions.ExecutionDeleteWeb, route_args, name='execution_delete'),
+        tornado.web.url(r'/executions/inspect/([0-9]+)', zoe_api.web.executions.ExecutionInspectWeb, route_args, name='execution_inspect')
     ]
 
     return web_routes

zoe_api/web/custom_request_handler.py

+# Copyright (c) 2014 thkang2
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Jinja2 templating for Tornado, taken from https://github.com/thkang2/jinja_tornado"""
+
+import json
+import datetime
+
+from jinja2 import Environment, FileSystemLoader, Markup
+
+from tornado.escape import squeeze, linkify, url_escape, xhtml_escape
+import tornado.web
+
+import zoe_lib.version
+
+import zoe_api.web.utils
+
+
+class JinjaApp(object):
+    """A Jinja2-capable Tornado application."""
+    def __init__(self, application, jinja_options=None):
+        self.application = application
+        self.jinja_options = jinja_options
+        self.init_app(application, jinja_options)
+
+    @classmethod
+    def init_app(cls, application, jinja_options=None):
+        """Init the application."""
+        app_settings = application.settings
+
+        _loader = FileSystemLoader(
+            app_settings.get('template_path', 'templates')
+        )
+
+        _jinja_config = {
+            'extensions': ['jinja2.ext.autoescape', 'jinja2.ext.with_'],
+            'auto_reload': app_settings.get('autoreload', False),
+            'loader': _loader,
+            'cache_size': 50 if app_settings.get('compiled_template_cache', True) else 0,
+            'autoescape': True if app_settings.get('autoescape', 'xhtml_escape') == "xhtml_escape" else False,
+        }
+
+        _jinja_config.update(**(jinja_options or {}))
+        environment = Environment(**_jinja_config)
+
+        application.jinja_environment = environment
+        app_settings['jinja_environment'] = environment
+        environment.filters.update(tojson=tojson_filter, xhtml_escape=xhtml_escape, url_escape=url_escape, squeeze=squeeze, linkify=linkify)
+
+        return environment
+
+
+def dumps(obj, **kwargs):
+    """Escape characters."""
+    # https://github.com/mitsuhiko/flask/blob/master/flask/json.py
+    return json.dumps(obj, **kwargs) \
+        .replace(u'<', u'\\u003c') \
+        .replace(u'>', u'\\u003e') \
+        .replace(u'&', u'\\u0026') \
+        .replace(u"'", u'\\u0027')
+
+
+def tojson_filter(obj, **kwargs):
+    """Filter for JSON output in templates."""
+    # https://github.com/mitsuhiko/flask/blob/master/flask/json.py
+    return Markup(dumps(obj, **kwargs))
+
+
+class ZoeRequestHandler(tornado.web.RequestHandler):
+    """usage:
+        class JinjaPoweredHandler(JinjaTemplateMixin, tornado.web.RequestHandler):
+            pass
+    """
+
+    def initialize(self, *args_, **kwargs_):
+        """Initialize the Jinja template system."""
+        # jinja environment is shared among an application
+        if 'jinja_environment' not in self.application.settings:
+            raise RuntimeError("Needs jinja2 Environment. Initialize with JinjaApp.init_app first")
+        else:
+            self._jinja_env = self.application.settings['jinja_environment']
+
+    def _render(self, template, **kwargs):
+        """ todo: support multiple template preprocessors """
+
+        ctx = {
+            'request': self.request,
+            'path_args': self.path_args,
+            'path_kwargs': self.path_kwargs,
+            'settings': self.application.settings,
+            'reverse_url': self.application.reverse_url,
+            'static_url': self.static_url,
+            'xsrf_form_html': self.xsrf_form_html,
+            'datetime': datetime,
+            'locale': self.locale,
+            'handler': self,
+            'zoe_version': zoe_lib.version.ZOE_VERSION
+        }
+
+        ctx.update(kwargs)
+        return template.render(ctx)
+
+    def render(self, template_name, **kwargs):
+        """ renders a template file. """
+        template = self._jinja_env.get_template(template_name)
+        try:
+            html = self._render(template, **kwargs)
+        except Exception:
+            zoe_api.web.utils.error_page(self, 'Jinja2 template exception', 500)
+            return
+        self.finish(html)
+
+    def render_string(self, source, **kwargs):
+        """ renders a template source string. """
+        template = self._jinja_env.from_string(source)
+        return self._render(template, **kwargs)
+
+    def data_received(self, chunk):
+        """Not implemented as we do not use stream uploads"""
+        pass

zoe_api/web/executions.py

@@ -17,93 +17,124 @@
 
 import json
 
-from flask import render_template, request, redirect, url_for, g
-
-from zoe_api.web.utils import get_auth, catch_exceptions
-import zoe_api.api_endpoint
 import zoe_api.exceptions
+from zoe_api.web.utils import get_auth, catch_exceptions
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
+from zoe_api.web.custom_request_handler import ZoeRequestHandler
+
+
+class ExecutionDefineWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
+
+    @catch_exceptions
+    def get(self):
+        """Define a new execution."""
+        get_auth(self)
+
+        self.render('execution_new.html')
+
+
+class ExecutionStartWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
+    @catch_exceptions
+    def post(self):
+        """Start an execution."""
+        uid, role = get_auth(self)
 
-@catch_exceptions
-def execution_define():
-    """Define a new execution."""
-    get_auth(request)
+        app_descr_json = self.request.files['file'][0]['body'].decode('utf-8')
+        app_descr = json.loads(app_descr_json)
+        exec_name = self.get_argument('exec_name')
 
-    return render_template('execution_new.html')
+        new_id = self.api_endpoint.execution_start(uid, role, exec_name, app_descr)
 
+        self.redirect(self.reverse_url('execution_inspect', new_id))
 
-@catch_exceptions
-def execution_start():
-    """Start an execution."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    app_descr_json = request.files['file'].read().decode('utf-8')
-    app_descr = json.loads(app_descr_json)
-    exec_name = request.form['exec_name']
+class ExecutionRestartWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
-    new_id = api_endpoint.execution_start(uid, role, exec_name, app_descr)
+    @catch_exceptions
+    def get(self, execution_id: int):
+        """Restart an already defined (and not running) execution."""
+        uid, role = get_auth(self)
 
-    return redirect(url_for('web.execution_inspect', execution_id=new_id))
+        e = self.api_endpoint.execution_by_id(uid, role, execution_id)
+        new_id = self.api_endpoint.execution_start(uid, role, e.name, e.description)
 
+        self.redirect(self.reverse_url('execution_inspect', new_id))
 
-@catch_exceptions
-def execution_restart(execution_id):
-    """Restart an already defined (and not running) execution."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
 
-    e = api_endpoint.execution_by_id(uid, role, execution_id)
-    new_id = api_endpoint.execution_start(uid, role, e.name, e.description)
+class ExecutionTerminateWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
-    return redirect(url_for('web.execution_inspect', execution_id=new_id))
+    @catch_exceptions
+    def get(self, execution_id: int):
+        """Terminate an execution."""
+        uid, role = get_auth(self)
 
+        success, message = self.api_endpoint.execution_terminate(uid, role, execution_id)
+        if not success:
+            raise zoe_api.exceptions.ZoeException(message)
 
-@catch_exceptions
-def execution_terminate(execution_id):
-    """Terminate an execution."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+        self.redirect(self.reverse_url('home_user'))
 
-    success, message = api_endpoint.execution_terminate(uid, role, execution_id)
-    if not success:
-        raise zoe_api.exceptions.ZoeException(message)
 
-    return redirect(url_for('web.home_user'))
+class ExecutionDeleteWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
+    @catch_exceptions
+    def get(self, execution_id: int):
+        """Delete an execution."""
+        uid, role = get_auth(self)
 
-@catch_exceptions
-def execution_delete(execution_id):
-    """Delete an execution."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+        success, message = self.api_endpoint.execution_delete(uid, role, execution_id)
+        if not success:
+            raise zoe_api.exceptions.ZoeException(message)
 
-    success, message = api_endpoint.execution_delete(uid, role, execution_id)
-    if not success:
-        raise zoe_api.exceptions.ZoeException(message)
+        self.redirect(self.reverse_url('home_user'))
 
-    return redirect(url_for('web.home_user'))
 
+class ExecutionInspectWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
-@catch_exceptions
-def execution_inspect(execution_id):
-    """Gather details about an execution."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+    @catch_exceptions
+    def get(self, execution_id):
+        """Gather details about an execution."""
+        uid, role = get_auth(self)
 
-    e = api_endpoint.execution_by_id(uid, role, execution_id)
+        e = self.api_endpoint.execution_by_id(uid, role, execution_id)
 
-    services_info = []
-    for service in e.services:
-        services_info.append(api_endpoint.service_by_id(uid, role, service.id))
+        services_info = []
+        for service in e.services:
+            services_info.append(self.api_endpoint.service_by_id(uid, role, service.id))
 
-    template_vars = {
-        "e": e,
-        "services_info": services_info
-    }
-    return render_template('execution_inspect.html', **template_vars)
+        template_vars = {
+            "e": e,
+            "services_info": services_info
+        }
+        self.render('execution_inspect.html', **template_vars)

zoe_api/web/start.py

@@ -18,57 +18,68 @@
 from random import randint
 import json
 
-from flask import render_template, request, g
-
-import zoe_api.api_endpoint
+from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
 from zoe_api.web.utils import get_auth, catch_exceptions
+from zoe_api.web.custom_request_handler import ZoeRequestHandler
+
 
+class RootWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""
+        super().initialize(**kwargs)
+        self.api_endpoint = kwargs['api_endpoint']  # type: APIEndpoint
 
-@catch_exceptions
-def index():
-    """Home page without authentication."""
-    return render_template('index.html')
+    @catch_exceptions
+    def get(self):
+        """Home page without authentication."""
+        self.render('index.html')
 
 
-@catch_exceptions
-def home_user():
-    """Home page with authentication."""
-    uid, role = get_auth(request)
-    api_endpoint = g.api_endpoint
-    assert isinstance(api_endpoint, zoe_api.api_endpoint.APIEndpoint)
+class HomeWeb(ZoeRequestHandler):
+    """Handler class"""
+    def initialize(self, **kwargs):
+        """Initializes the request handler."""