Cleanups and fixes

caaas/api.py

@@ -18,30 +18,6 @@ def api_status():
     return jsonify(**data)
 
 
-@app.route("/api/full-status")
-def api_full_status():
-    db = CAaaState()
-    data = {}
-    user_list = db.get_all_users()
-    for user_id, username in user_list:
-        data[username] = {}
-        data[username]["num_clusters"] = db.count_clusters(user_id)
-        data[username]["num_containers"] = db.count_containers(user_id)
-        data[username]["has_notebook"] = db.has_notebook(user_id)
-    return jsonify(**data)
-
-
-@app.route("/api/<username>/status")
-def api_user_status(username):
-    db = CAaaState()
-    user_id = db.get_user_id(username)
-    cluster_list = db.get_clusters(user_id)
-    for clid in cluster_list:
-        cluster_list[clid]["is_notebook"] = cluster_list[clid]["name"] == "notebook"
-        cluster_list[clid]["num_containers"] = db.count_containers(user_id, clid)
-    return jsonify(**cluster_list)
-
-
 @app.route("/api/<username>/cluster/<cluster_id>/terminate")
 def api_terminate_cluster(username, cluster_id):
     db = CAaaState()

caaas/sql.py

@@ -352,6 +352,20 @@ class CAaaState:
         cursor.execute(q, (app_id,))
         self._close_cursor(cursor)
 
+    def get_applications(self, user_id=None) -> dict:
+        cursor = self._get_cursor(dictionary=True)
+        if user_id is None:
+            q = "SELECT * FROM applications"
+            cursor.execute(q)
+        else:
+            q = "SELECT * FROM applications WHERE user_id=%s"
+            cursor.execute(q, (user_id,))
+        res = []
+        for row in cursor:
+            res.append((dict(row)))
+        self._close_cursor(cursor)
+        return res
+
     def get_application(self, app_id: int) -> dict:
         cursor = self._get_cursor(dictionary=True)
         q = "SELECT * FROM applications WHERE id=%s"

caaas/swarm_manager.py

@@ -28,6 +28,7 @@ class ContainerOptions:
         self.volume_binds = []
         self.volumes = []
         self.command = ""
+        self.memory_limit = '2g'
 
     def add_env_variable(self, name, value):
         self.env[name] = value
@@ -51,6 +52,12 @@ class ContainerOptions:
     def get_command(self):
         return self.command
 
+    def set_memory_limit(self, limit):
+        self.memory_limit = limit
+
+    def get_memory_limit(self):
+        return self.memory_limit
+
 
 class SwarmManager:
     def __init__(self):
@@ -127,6 +134,7 @@ class SwarmManager:
         options.add_env_variable("SPARK_MASTER_IP", master_info["docker_ip"])
         options.add_env_variable("SPARK_WORKER_RAM", cluster_descr.executor_ram_size)
         options.add_env_variable("SPARK_WORKER_CORES", cluster_descr.worker_cores)
+        options.set_memory_limit(cluster_descr.executor_ram_size)
         info = self._spawn_container(WORKER_IMAGE, options)
         cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-worker-" + str(count))
         db.new_proxy_entry(get_uuid(), cluster_id, "http://" + info["docker_ip"] + ":8081", "web interface", cont_id)
@@ -139,6 +147,7 @@ class SwarmManager:
         options.add_env_variable("SPARK_MASTER_IP", master_info["docker_ip"])
         options.add_env_variable("SPARK_EXECUTOR_RAM", cluster_descr.executor_ram_size)
         options.add_env_variable("PROXY_ID", proxy_id)
+        options.set_memory_limit(cluster_descr.executor_ram_size)
         info = self._spawn_container(NOTEBOOK_IMAGE, options)
         cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-notebook")
         db.new_proxy_entry(proxy_id, cluster_id, "http://" + info["docker_ip"] + ":9000/proxy/" + proxy_id, "notebook", cont_id)
@@ -155,13 +164,16 @@ class SwarmManager:
         options.add_env_variable("SPARK_OPTIONS", app["spark_options"])
         options.add_volume_bind(app["path"], '/app', True)
         options.set_command("/opt/submit.sh /app " + app["cmd"])
+        options.set_memory_limit(cluster_descr.executor_ram_size)
         info = self._spawn_container(SUBMIT_IMAGE, options)
         cont_id = state.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-submit")
         state.new_proxy_entry(get_uuid(), cluster_id, "http://" + info["docker_ip"] + ":4040", "spark application", cont_id)
         return info
 
     def _spawn_container(self, image, options):
-        host_config = create_host_config(network_mode="bridge", binds=options.get_volume_binds())
+        host_config = create_host_config(network_mode="bridge",
+                                         binds=options.get_volume_binds(),
+                                         mem_limit=options.get_memory_limit())
         cont = self.cli.create_container(image=image,
                                          environment=options.get_environment(),
                                          network_disabled=False,
@@ -218,3 +230,6 @@ class SwarmManager:
             return False
         ret = self.cli.inspect_container(container=cont["docker_id"])
         return ret["State"]["Running"]
+
+    def swarm_status(self):
+        return self.cli.info()

caaas/templates/apps.html

+{% extends "base_user.html" %}
+{% block title %}{{ user }} status{% endblock %}
+{% block content %}
+<h2>Status overview for user {{ user }}</h2>
+    {% if apps|length == 0 and not has_notebook %}
+        <p>You have no clusters running at this time</p>
+    {% endif %}
+
+    {% if has_notebook %}
+        <p>You have a notebook running:
+        <a href="{{ notebook_address }}">open</a>
+        <a href="{{ url_for("web_terminate", username=user, cluster_id=notebook_cluster_id) }}">terminate</a>
+        </p>
+    {% endif %}
+
+    <table id="app_list">
+    <thead>
+    <tr>
+        <th>Name</th>
+        <th>Command line</th>
+        <th>Spark options</th>
+        <th>Started</th>
+        <th>Finished</th>
+        <th>Status</th>
+        <th>Logs</th>
+    </tr>
+    </thead>
+        {% for a in apps %}
+            <tr>
+            <td>{{ a["execution_name"] }}</td>
+            <td>{{ a["cmd"] }}</td>
+            <td>{{ a["spark_options"] }}</td>
+            <td>{{ a["time_started"] }}</td>
+            <td>{{ a["time_finished"] }}</td>
+            <td>{{ a["status"] }}</td>
+            <td>
+                {% if a["status"] == "finished" %}
+                <a href="{{ url_for("api_history_log_archive", username=user, app_id=a["id"]) }}">Zip</a>
+                {% endif %}
+                {% if a["status"] == "running" %}
+                <a href="{{ url_for("web_inspect", username=user, app_id=a["cluster_id"]) }}">Inspect</a>
+                {% endif %}
+            </td>
+            <td>
+                {% if a["status"] == "running" %}
+                {{ url_for("web_terminate", username=user, cluster_id=a["cluster_id"]) }}
+                {% endif %}
+            </td>
+            </tr>
+            <!-- FIXME: add a re-run link -->
+        {% endfor %}
+    </table>
+{% endblock %}
\ No newline at end of file

caaas/templates/base.html

@@ -17,6 +17,9 @@
                 .done(function( data ) {
                     $("#num_nodes").text(data.num_nodes);
                     $("#num_containers").text(data.num_containers);
+                }).error(function( data ) {
+                    $("#num_nodes").text("N/A");
+                    $("#num_containers").text("N/A");
                 });
         }
         update_status();
@@ -26,7 +29,7 @@
     <div id="content">{% block content %}{% endblock %}</div>
     <div id="footer">
         {% block footer %}
-            <div class="caaas_status"><p><em>CAaaS</em> status: <span id="num_nodes">0</span> available nodes, <span id="num_containers">0</span> containers running</p></div>
+            <div class="caaas_status"><p><em>CAaaS</em> status: <span id="num_nodes">N/A</span> available nodes, <span id="num_containers">N/A</span> containers running</p></div>
             <h6>&copy; Copyright 2015 by <a href="http://distsysgroup.wordpress.com/">DSG</a></h6>
         {% endblock %}
     </div>

caaas/templates/home.html

@@ -3,18 +3,18 @@
 {% block content %}
 <p>Start an activity:</p>
 <ul>
-    <li><a href="/web/{{ user }}/spark-notebook">Give me a Spark notebook</a></li>
-    <li><a href="/web/{{ user }}/submit-spark-app">Submit a Spark application</a></li>
+    <li><a href="{{ url_for("web_notebook", username=user) }}">Give me a Spark notebook</a></li>
+    <li><a href="{{ url_for("web_spark_submit", username=user) }}">Submit a Spark application</a></li>
 </ul>
 
-<p>Monitor my activities:</p>
+<p>My activities:</p>
 <ul>
-    <li><a href="/web/{{ user }}/status">Running applications</a></li>
+    <li><a href="{{ url_for("web_user_apps", username=user) }}">Applications</a></li>
 </ul>
 
 <p>Cluster status:</p>
 <ul>
-    <li><a href="/web/status">Overview</a></li>
+    <li><a href="{{ url_for("web_status") }}">Overview</a></li>
 </ul>
 
 

caaas/templates/status.html

 {% extends "base_user.html" %}
 {% block title %}Status{% endblock %}
 {% block content %}
-<script type="application/javascript">
-    function gen_one_user(i, item) {
-        var block = $("<div></div>");
-        block.append("<p></p>").text(i);
-        var ul = $("<ul></ul>");
-        ul.append($("<li></li>").text("Num. clusters: " + item.num_clusters));
-        ul.append($("<li></li>").text("Num. containers: " + item.num_containers));
-        ul.append($("<li></li>").text("Has a notebook: " + item.has_notebook));
-        block.append(ul);
-        return block;
-    }
-    (function() {
-        $.getJSON( api_endpoint + "/full-status")
-                .done(function( data ) {
-                    $.each(data, function(i, item) {
-                        $("#userlist").append(gen_one_user(i, item))
-                    })
-                });
-    })();
-</script>
-
 <h2>System status overview</h2>
-
+    <ul>
+    <li>Containers: {{ Containers }}</li>
+    <li>Images: {{ Images }}</li>
+    <li>MemTotal: {{ MemTotal }}</li>
+    <li>Memory limit: {{ MemoryLimit }}</li>
+    <li>NCPU: {{ NCPU }}</li>
+    <li>Driver status:</li>
+    <ul>
+        {% for entry in DriverStatus %}
+            <li>{{ entry[0] }}: {{ entry[1] }}</li>
+        {% endfor %}
+    </ul>
+    </ul>
 <div id="userlist"></div>
 {% endblock %}
\ No newline at end of file

caaas/templates/user-status.html

-{% extends "base_user.html" %}
-{% block title %}{{ user }} status{% endblock %}
-{% block content %}
-<h2>Status overview for user {{ user }}</h2>
-
-<div id="cluster_list">
-{% if clusters|length == 0 %}
-    <p>You have no clusters running at this time</p>
-{% endif %}
-
-<ul>
-{% for c in clusters %}
-    <li>{{ clusters[c]["name"] }} (<a href="/web/{{ user }}/cluster/{{ c }}/inspect">inspect</a>,
-        <a href="/web/{{ user }}/cluster/{{ c }}/terminate">terminate</a>)</li>
-{% endfor %}
-</ul>
-</div>
-{% endblock %}
\ No newline at end of file

caaas/web.py

@@ -11,7 +11,8 @@ def index():
 
 @app.route("/web/status")
 def web_status():
-    return render_template('status.html')
+    status = sm.swarm_status()
+    return render_template('status.html', **status)
 
 
 @app.route("/web/<username>")
@@ -24,19 +25,19 @@ def web_index(username):
     return render_template('home.html', **template_vars)
 
 
-@app.route("/web/<username>/status")
-def web_user_status(username):
-    db = CAaaState()
-    user_id = db.get_user_id(username)
-    cluster_list = db.get_clusters(user_id)
-    for clid in cluster_list:
-        cluster_list[clid]["is_notebook"] = cluster_list[clid]["name"] == "notebook"
-        cluster_list[clid]["num_containers"] = db.count_containers(user_id, clid)
+@app.route("/web/<username>/apps")
+def web_user_apps(username):
+    state = CAaaState()
+    user_id = state.get_user_id(username)
+    apps = state.get_applications(user_id)
     template_vars = {
         "user": username,
-        "clusters": cluster_list
+        "apps": apps,
+        "has_notebook": state.has_notebook(user_id),
+        "notebook_address": sm.get_notebook(user_id),
+        "notebook_cluster_id": state.get_notebook(user_id)
     }
-    return render_template('user-status.html', **template_vars)
+    return render_template('apps.html', **template_vars)
 
 
 @app.route("/web/<username>/spark-notebook")