Skip to content
GitLab
Commit dd9226c8 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Cleanups and fixes

parent c177a3fc
Expand all Hide whitespace changes
Inline Side-by-side
.idea/workspace.xml
View file @ dd9226c8
This diff is collapsed.
caaas/api.py
View file @ dd9226c8
......@@ -18,30 +18,6 @@ def api_status():
return jsonify(**data)
@app.route("/api/full-status")
def api_full_status():
db = CAaaState()
data = {}
user_list = db.get_all_users()
for user_id, username in user_list:
data[username] = {}
data[username]["num_clusters"] = db.count_clusters(user_id)
data[username]["num_containers"] = db.count_containers(user_id)
data[username]["has_notebook"] = db.has_notebook(user_id)
return jsonify(**data)
@app.route("/api/<username>/status")
def api_user_status(username):
db = CAaaState()
user_id = db.get_user_id(username)
cluster_list = db.get_clusters(user_id)
for clid in cluster_list:
cluster_list[clid]["is_notebook"] = cluster_list[clid]["name"] == "notebook"
cluster_list[clid]["num_containers"] = db.count_containers(user_id, clid)
return jsonify(**cluster_list)
@app.route("/api/<username>/cluster/<cluster_id>/terminate")
def api_terminate_cluster(username, cluster_id):
db = CAaaState()
......
caaas/sql.py
View file @ dd9226c8
......@@ -352,6 +352,20 @@ class CAaaState:
cursor.execute(q, (app_id,))
self._close_cursor(cursor)
def get_applications(self, user_id=None) -> dict:
cursor = self._get_cursor(dictionary=True)
if user_id is None:
q = "SELECT * FROM applications"
cursor.execute(q)
else:
q = "SELECT * FROM applications WHERE user_id=%s"
cursor.execute(q, (user_id,))
res = []
for row in cursor:
res.append((dict(row)))
self._close_cursor(cursor)
return res
def get_application(self, app_id: int) -> dict:
cursor = self._get_cursor(dictionary=True)
q = "SELECT * FROM applications WHERE id=%s"
......
caaas/swarm_manager.py
View file @ dd9226c8
......@@ -28,6 +28,7 @@ class ContainerOptions:
self.volume_binds = []
self.volumes = []
self.command = ""
self.memory_limit = '2g'
def add_env_variable(self, name, value):
self.env[name] = value
......@@ -51,6 +52,12 @@ class ContainerOptions:
def get_command(self):
return self.command
def set_memory_limit(self, limit):
self.memory_limit = limit
def get_memory_limit(self):
return self.memory_limit
class SwarmManager:
def __init__(self):
......@@ -127,6 +134,7 @@ class SwarmManager:
options.add_env_variable("SPARK_MASTER_IP", master_info["docker_ip"])
options.add_env_variable("SPARK_WORKER_RAM", cluster_descr.executor_ram_size)
options.add_env_variable("SPARK_WORKER_CORES", cluster_descr.worker_cores)
options.set_memory_limit(cluster_descr.executor_ram_size)
info = self._spawn_container(WORKER_IMAGE, options)
cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-worker-" + str(count))
db.new_proxy_entry(get_uuid(), cluster_id, "http://" + info["docker_ip"] + ":8081", "web interface", cont_id)
......@@ -139,6 +147,7 @@ class SwarmManager:
options.add_env_variable("SPARK_MASTER_IP", master_info["docker_ip"])
options.add_env_variable("SPARK_EXECUTOR_RAM", cluster_descr.executor_ram_size)
options.add_env_variable("PROXY_ID", proxy_id)
options.set_memory_limit(cluster_descr.executor_ram_size)
info = self._spawn_container(NOTEBOOK_IMAGE, options)
cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-notebook")
db.new_proxy_entry(proxy_id, cluster_id, "http://" + info["docker_ip"] + ":9000/proxy/" + proxy_id, "notebook", cont_id)
......@@ -155,13 +164,16 @@ class SwarmManager:
options.add_env_variable("SPARK_OPTIONS", app["spark_options"])
options.add_volume_bind(app["path"], '/app', True)
options.set_command("/opt/submit.sh /app " + app["cmd"])
options.set_memory_limit(cluster_descr.executor_ram_size)
info = self._spawn_container(SUBMIT_IMAGE, options)
cont_id = state.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-submit")
state.new_proxy_entry(get_uuid(), cluster_id, "http://" + info["docker_ip"] + ":4040", "spark application", cont_id)
return info
def _spawn_container(self, image, options):
host_config = create_host_config(network_mode="bridge", binds=options.get_volume_binds())
host_config = create_host_config(network_mode="bridge",
binds=options.get_volume_binds(),
mem_limit=options.get_memory_limit())
cont = self.cli.create_container(image=image,
environment=options.get_environment(),
network_disabled=False,
......@@ -218,3 +230,6 @@ class SwarmManager:
return False
ret = self.cli.inspect_container(container=cont["docker_id"])
return ret["State"]["Running"]
def swarm_status(self):
return self.cli.info()
caaas/templates/apps.html 0 → 100644
View file @ dd9226c8
{% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %}
{% block content %}
<h2>Status overview for user {{ user }}</h2>
{% if apps|length == 0 and not has_notebook %}
<p>You have no clusters running at this time</p>
{% endif %}
{% if has_notebook %}
<p>You have a notebook running:
<a href="{{ notebook_address }}">open</a>
<a href="{{ url_for("web_terminate", username=user, cluster_id=notebook_cluster_id) }}">terminate</a>
</p>
{% endif %}
<table id="app_list">
<thead>
<tr>
<th>Name</th>
<th>Command line</th>
<th>Spark options</th>
<th>Started</th>
<th>Finished</th>
<th>Status</th>
<th>Logs</th>
</tr>
</thead>
{% for a in apps %}
<tr>
<td>{{ a["execution_name"] }}</td>
<td>{{ a["cmd"] }}</td>
<td>{{ a["spark_options"] }}</td>
<td>{{ a["time_started"] }}</td>
<td>{{ a["time_finished"] }}</td>
<td>{{ a["status"] }}</td>
<td>
{% if a["status"] == "finished" %}
<a href="{{ url_for("api_history_log_archive", username=user, app_id=a["id"]) }}">Zip</a>
{% endif %}
{% if a["status"] == "running" %}
<a href="{{ url_for("web_inspect", username=user, app_id=a["cluster_id"]) }}">Inspect</a>
{% endif %}
</td>
<td>
{% if a["status"] == "running" %}
{{ url_for("web_terminate", username=user, cluster_id=a["cluster_id"]) }}
{% endif %}
</td>
</tr>
<!-- FIXME: add a re-run link -->
{% endfor %}
</table>
{% endblock %}
\ No newline at end of file
caaas/templates/base.html
View file @ dd9226c8
......@@ -17,6 +17,9 @@
.done(function( data ) {
$("#num_nodes").text(data.num_nodes);
$("#num_containers").text(data.num_containers);
}).error(function( data ) {
$("#num_nodes").text("N/A");
$("#num_containers").text("N/A");
});
}
update_status();
......@@ -26,7 +29,7 @@
<div id="content">{% block content %}{% endblock %}</div>
<div id="footer">
{% block footer %}
<div class="caaas_status"><p><em>CAaaS</em> status: <span id="num_nodes">0</span> available nodes, <span id="num_containers">0</span> containers running</p></div>
<div class="caaas_status"><p><em>CAaaS</em> status: <span id="num_nodes">N/A</span> available nodes, <span id="num_containers">N/A</span> containers running</p></div>
<h6>&copy; Copyright 2015 by <a href="http://distsysgroup.wordpress.com/">DSG</a></h6>
{% endblock %}
</div>
......
caaas/templates/home.html
View file @ dd9226c8
......@@ -3,18 +3,18 @@
{% block content %}
<p>Start an activity:</p>
<ul>
<li><a href="/web/{{ user }}/spark-notebook">Give me a Spark notebook</a></li>
<li><a href="/web/{{ user }}/submit-spark-app">Submit a Spark application</a></li>
<li><a href="{{ url_for("web_notebook", username=user) }}">Give me a Spark notebook</a></li>
<li><a href="{{ url_for("web_spark_submit", username=user) }}">Submit a Spark application</a></li>
</ul>
<p>Monitor my activities:</p>
<p>My activities:</p>
<ul>
<li><a href="/web/{{ user }}/status">Running applications</a></li>
<li><a href="{{ url_for("web_user_apps", username=user) }}">Applications</a></li>
</ul>
<p>Cluster status:</p>
<ul>
<li><a href="/web/status">Overview</a></li>
<li><a href="{{ url_for("web_status") }}">Overview</a></li>
</ul>
......
caaas/templates/status.html
View file @ dd9226c8
{% extends "base_user.html" %}
{% block title %}Status{% endblock %}
{% block content %}
<script type="application/javascript">
function gen_one_user(i, item) {
var block = $("<div></div>");
block.append("<p></p>").text(i);
var ul = $("<ul></ul>");
ul.append($("<li></li>").text("Num. clusters: " + item.num_clusters));
ul.append($("<li></li>").text("Num. containers: " + item.num_containers));
ul.append($("<li></li>").text("Has a notebook: " + item.has_notebook));
block.append(ul);
return block;
}
(function() {
$.getJSON( api_endpoint + "/full-status")
.done(function( data ) {
$.each(data, function(i, item) {
$("#userlist").append(gen_one_user(i, item))
})
});
})();
</script>
<h2>System status overview</h2>
<ul>
<li>Containers: {{ Containers }}</li>
<li>Images: {{ Images }}</li>
<li>MemTotal: {{ MemTotal }}</li>
<li>Memory limit: {{ MemoryLimit }}</li>
<li>NCPU: {{ NCPU }}</li>
<li>Driver status:</li>
<ul>
{% for entry in DriverStatus %}
<li>{{ entry[0] }}: {{ entry[1] }}</li>
{% endfor %}
</ul>
</ul>
<div id="userlist"></div>
{% endblock %}
\ No newline at end of file
caaas/templates/user-status.html deleted 100644 → 0
View file @ c177a3fc
{% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %}
{% block content %}
<h2>Status overview for user {{ user }}</h2>
<div id="cluster_list">
{% if clusters|length == 0 %}
<p>You have no clusters running at this time</p>
{% endif %}
<ul>
{% for c in clusters %}
<li>{{ clusters[c]["name"] }} (<a href="/web/{{ user }}/cluster/{{ c }}/inspect">inspect</a>,
<a href="/web/{{ user }}/cluster/{{ c }}/terminate">terminate</a>)</li>
{% endfor %}
</ul>
</div>
{% endblock %}
\ No newline at end of file
caaas/web.py
View file @ dd9226c8
......@@ -11,7 +11,8 @@ def index():
@app.route("/web/status")
def web_status():
return render_template('status.html')
status = sm.swarm_status()
return render_template('status.html', **status)
@app.route("/web/<username>")
......@@ -24,19 +25,19 @@ def web_index(username):
return render_template('home.html', **template_vars)
@app.route("/web/<username>/status")
def web_user_status(username):
db = CAaaState()
user_id = db.get_user_id(username)
cluster_list = db.get_clusters(user_id)
for clid in cluster_list:
cluster_list[clid]["is_notebook"] = cluster_list[clid]["name"] == "notebook"
cluster_list[clid]["num_containers"] = db.count_containers(user_id, clid)
@app.route("/web/<username>/apps")
def web_user_apps(username):
state = CAaaState()
user_id = state.get_user_id(username)
apps = state.get_applications(user_id)
template_vars = {
"user": username,
"clusters": cluster_list
"apps": apps,
"has_notebook": state.has_notebook(user_id),
"notebook_address": sm.get_notebook(user_id),
"notebook_cluster_id": state.get_notebook(user_id)
}
return render_template('user-status.html', **template_vars)
return render_template('apps.html', **template_vars)
@app.route("/web/<username>/spark-notebook")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment