Start refactoring web interface for v2

common/state/application.py

@@ -110,4 +110,13 @@ class SparkSubmitApplication(SparkApplication):
 
 
 class PlainApplication:
-    pass
+    id = None
+    name = None
+    required_resources = None
+    user_id = None
+    executions = None
+    type = None
+    master_image = None
+    worker_image = None
+    notebook_image = None
+    submit_image = None

common/state/execution.py

@@ -84,4 +84,16 @@ class SparkSubmitExecution(Execution):
 
 
 class PlainExecution:
-    pass
+    id = None
+    name = None
+    assigned_resources = None
+    application_id = None
+    time_started = None
+    time_scheduled = None
+    time_finished = None
+    status = None
+    termination_notice = None
+    cluster_id = None
+    type = None
+    commandline = None
+    spark_opts = None

common/state/user.py

@@ -24,4 +24,6 @@ class User(Base):
 
 
 class PlainUser:
-    pass
+    id = None
+    email = None
+

zoe_client/client.py

@@ -140,7 +140,7 @@ class ZoeClient:
             return None
         return self.server.application_status(application.id)
 
-    def spark_application_list(self, user_id) -> [PlainApplication]:
+    def application_list(self, user_id) -> [PlainApplication]:
         try:
             self.state.query(User).filter_by(id=user_id).one()
         except NoResultFound:

zoe_web/__init__.py

 from flask import Flask
+from zoe_web.api import api
+from zoe_web.web import web
+
 app = Flask(__name__)
+app.register_blueprint(web, url_prefix='/web')
+app.register_blueprint(api, url_prefix='/web/api')
 
-import zoe_web.web
-import zoe_web.api
+app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"

zoe_web/api/__init__.py

+from flask import Blueprint, abort
+
+api = Blueprint('api', __name__)
+
+
+@api.route('/status/basic')
+def basic_status():
+    abort(404)

zoe_web/api.py → zoe_web/api/api.py

-from flask import jsonify, request, send_file, abort
+from flask import jsonify, request, send_file, abort, Blueprint
 from zipfile import is_zipfile
 
-from zoe_web import app
-from zoe_web.sql import CAaaState
-from zoe_web.spark_app_execution import application_submitted, setup_volume, AppHistory
-from zoe_web.swarm_manager import sm
 
-STATS_CACHING_EXPIRATION = 1  # seconds
 
+zoeweb_api = Blueprint('zoeweb_api', __name__)
 
 @app.route("/api/<int:user_id>/cluster/<int:cluster_id>/terminate")
 def api_terminate_cluster(user_id, cluster_id):

zoe_web/spark_app_execution.py

-from zipfile import ZipFile
-import os
-import shutil
-
-from zoe_web.sql import CAaaState
-from zoe_web.config_parser import config
-
-
-class AppHistory:
-    def __init__(self, user_id):
-        self.base_path = config.history_storage_path
-        self.per_user_max_count = int(config.history_per_user_count)
-        self.user_id = str(user_id)
-
-    def _app_path(self, app_id):
-        return os.path.join(self.base_path, self.user_id, str(app_id))
-
-    def _delete_app_history(self, app_id):
-        app_path = self._app_path(app_id)
-        shutil.rmtree(app_path)
-
-    def cleanup(self):
-        state = CAaaState()
-        num_apps = state.count_apps_finished(self.user_id)
-        if num_apps > self.per_user_max_count:
-            app_id = state.remove_oldest_application(self.user_id)
-            self._delete_app_history(app_id)
-
-    def add_application_zip(self, app_id, file_data):
-        app_path = self._app_path(app_id)
-        if not os.path.exists(app_path):
-            os.makedirs(app_path)
-        file_data.save(os.path.join(app_path, "app.zip"))
-
-    def save_log(self, app_id, logname, log):
-        app_path = self._app_path(app_id)
-        assert os.path.exists(app_path)
-        zip_path = os.path.join(app_path, "logs.zip")
-        z = ZipFile(zip_path, mode="a")
-        z.writestr(logname + ".txt", log)
-        z.close()
-
-    def get_log_archive_path(self, app_id):
-        app_path = self._app_path(app_id)
-        zip_path = os.path.join(app_path, "logs.zip")
-        if not os.path.exists(app_path):
-            return None
-        else:
-            return zip_path
-
-
-def application_submitted(user_id, execution_name, spark_options, commandline, file_data) -> int:
-    ah = AppHistory(user_id)
-    ah.cleanup()
-    state = CAaaState()
-    app_id = state.new_application(user_id, execution_name, spark_options, commandline)
-    ah.add_application_zip(app_id, file_data)
-    return app_id
-
-
-def setup_volume(user_id, app_id, app_pkg):
-    app_pkg = ZipFile(app_pkg)
-    exec_path = config.docker_volume_path
-    exec_path = os.path.join(exec_path, str(user_id), str(app_id))
-    os.makedirs(exec_path)
-    app_pkg.extractall(exec_path)
-    state = CAaaState()
-    state.application_ready(app_id)

zoe_web/utils.py

-from uuid import uuid1 as uuid
-
-
-def get_uuid():
-    return str(uuid())

zoe_web/web/__init__.py

+from flask import Blueprint, abort, render_template
+
+from zoe_client import ZoeClient
+import zoe_web.web.utils as web_utils
+
+web = Blueprint('web', __name__, template_folder='templates', static_folder='static')
+
+
+@web.route('/')
+def index():
+    return render_template('index.html')
+
+
+@web.route('/home')
+def home():
+    client = ZoeClient()
+    user = web_utils.check_user(client)
+    template_vars = {
+        "user_id": user.id,
+        "email": user.email
+    }
+    return render_template('home.html', **template_vars)

zoe_web/templates/base.html → zoe_web/web/templates/base.html

@@ -3,12 +3,9 @@
 <head>
     {% block head %}
     <meta charset="UTF-8">
-    <title>{% block title %}{% endblock %} - CAAAS</title>
+    <title>{% block title %}{% endblock %} - Zoe</title>
     <script src="/static/jquery-2.1.4.min.js" type="application/javascript"></script>
-    <script type="application/javascript">
-        var api_endpoint = "/api";
-    </script>
-    <link rel="stylesheet" href="/static/caaas.css" type="text/css">
+    <link rel="stylesheet" href="/static/zoe.css" type="text/css">
     {% endblock %}
 </head>
 <body>
@@ -26,13 +23,13 @@
         update_status();
         window.setInterval(update_status, 5000);
     </script>
-    <h1>DSG CAaaS system - Analytics on demand</h1>
+    <h1>Zoe - Analytics on demand</h1>
     <div id="content">{% block content %}{% endblock %}</div>
     <div id="footer">
         {% block footer %}
-            <div class="caaas_status_line">
+            <div class="status_line">
                 <p>
-                    <span id="app_name"><a href="{{ url_for("web_status") }}">CAaaS</a></span>
+                    <span id="app_name"><a href="{{ url_for("web_status") }}">Zoe</a></span>
                     <span id="status">
                         [<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers]
                     </span>

zoe_web/web/utils.py

+from flask import session, redirect, url_for
+
+from zoe_client import ZoeClient
+from zoe_web.web import web
+
+
+def check_user(zoeclient: ZoeClient):
+    if 'user_id' not in session:
+        return redirect(url_for(web.index))
+    if not zoeclient.user_check(session['user_id']):
+        return redirect(url_for('index'))

zoe_web/web.py → zoe_web/web/web.py

@@ -2,11 +2,7 @@ from flask import render_template, redirect, url_for, abort
 
 from zoe_web import app
 from zoe_client import ZoeClient
-
-
-@app.route("/web/")
-def index():
-    return render_template('index.html')
+from common.configuration import conf
 
 
 @app.route("/web/status")
@@ -32,7 +28,7 @@ def web_index(user_id):
         return redirect(url_for('index'))
     template_vars = {
         "user_id": user_id,
-        "email": client.user_get_email(user_id)
+        "email": client.user_get(user_id).email
     }
     return render_template('home.html', **template_vars)
 
@@ -43,7 +39,7 @@ def web_user_apps(user_id):
     if not client.user_check(user_id):
         return redirect(url_for('index'))
 
-    apps = client.spark_application_list(user_id)
+    apps = client.application_list(user_id)
     template_vars = {
         "user_id": user_id,
         "apps": apps
@@ -53,53 +49,48 @@ def web_user_apps(user_id):
 
 @app.route("/web/<int:user_id>/spark-notebook")
 def web_notebook(user_id):
-    state = CAaaState()
-    if not state.check_user_id(user_id):
+    client = ZoeClient()
+    if not client.user_check(user_id):
         return redirect(url_for('index'))
 
     template_vars = {
         "user_id": user_id,
-        "notebook_address": sm.get_notebook(user_id),
-        "max_age": config.cleanup_notebooks_older_than,
-        "wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning)
+        "max_age": conf['notebook_max_age_no_activity'],
+        "wrn_time": conf['notebook_max_age_no_activity'] - conf['notebook_warning_age_no_activity']
     }
     return render_template('notebook.html', **template_vars)
 
 
-@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/inspect")
-def web_inspect(user_id, cluster_id):
-    state = CAaaState()
-    if not state.check_user_id(user_id):
+@app.route("/web/<int:user_id>/cluster/<int:app_id>/inspect")
+def web_inspect(user_id, app_id):
+    client = ZoeClient()
+    if not client.user_check(user_id):
         return redirect(url_for('index'))
 
-    cluster = state.get_cluster(cluster_id)
-    if cluster["user_id"] != user_id:
+    application = client.application_get(app_id)
+    if application.user_id != user_id:
         abort(404)
-    containers = state.get_containers(cluster_id=cluster_id)
-    clist = []
-    for cid, cinfo in containers.items():
-        plist = get_container_addresses(cid)
-        clist.append([cinfo["contents"], plist, cid])
+    app_report = client.application_status(app_id)
     template_vars = {
-        "cluster_name": cluster["name"],
-        "containers": clist,
+        "application": app_report,
         "user_id": user_id
     }
     return render_template('inspect.html', **template_vars)
 
 
-@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/terminate")
-def web_terminate(user_id, cluster_id):
-    state = CAaaState()
-    if not state.check_user_id(user_id):
+@app.route("/web/<int:user_id>/execution/<int:execution_id>/terminate")
+def web_terminate(user_id, execution_id):
+    client = ZoeClient()
+    if not client.user_check(user_id):
         return redirect(url_for('index'))
 
-    cluster = state.get_cluster(cluster_id)
-    if cluster["user_id"] != user_id:
+    execution = client.execution_get(execution_id)
+    application = client.application_get(execution.application_id)
+    if application.user_id != user_id:
         abort(404)
     template_vars = {
-        "cluster_name": cluster["name"],
-        "cluster_id": cluster_id,
+        "execution_name": execution.name,
+        "execution_id": execution_id,
         "user_id": user_id
     }
     return render_template('terminate.html', **template_vars)
@@ -107,19 +98,18 @@ def web_terminate(user_id, cluster_id):
 
 @app.route("/web/<int:user_id>/container/<int:container_id>/logs")
 def web_logs(user_id, container_id):
-    state = CAaaState()
-    if not state.check_user_id(user_id):
+    client = ZoeClient()
+    if not client.user_check(user_id):
         return redirect(url_for('index'))
 
-    cont = state.get_container(container_id)
+    cont = client.container_get(container_id)
     if user_id != cont["user_id"]:
         abort(404)
 
-    logs = sm.get_log(container_id)
+    logs = client.log_get(container_id)
     if logs is None:
         abort(404)
     else:
-        logs = logs.decode("ascii")
         ret = {
             'user_id': user_id,
             'cont_contents': cont['contents'],
@@ -130,8 +120,8 @@ def web_logs(user_id, container_id):
 
 @app.route("/web/<int:user_id>/submit-spark-app")
 def web_spark_submit(user_id):
-    state = CAaaState()
-    if not state.check_user_id(user_id):
+    client = ZoeClient()
+    if not client.user_check(user_id):
         return redirect(url_for('index'))
 
     template_vars = {

zoectl.py

@@ -75,7 +75,7 @@ def app_rm_cmd(args):
             e = client.execution_get(eid)
             if e.status == "running":
                 print("Terminating execution {}".format(e.name))
-                client.execution_terminate(e)
+                client.execution_terminate(e.id)
 
     client.application_remove(application.id)
 
@@ -92,7 +92,7 @@ def app_inspect_cmd(args):
 
 def app_list_cmd(args):
     client = ZoeClient()
-    applications = client.spark_application_list(args.id)
+    applications = client.application_list(args.id)
     if len(applications) > 0:
         print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
     for app in applications: