Commit ca9a4ef9 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Use user_id also in web urls

parent 899db8e5
This diff is collapsed.
...@@ -3,5 +3,3 @@ app = Flask(__name__) ...@@ -3,5 +3,3 @@ app = Flask(__name__)
import caaas.web import caaas.web
import caaas.api import caaas.api
...@@ -10,6 +10,15 @@ from caaas.swarm_manager import sm ...@@ -10,6 +10,15 @@ from caaas.swarm_manager import sm
STATS_CACHING_EXPIRATION = 1 # seconds STATS_CACHING_EXPIRATION = 1 # seconds
@app.route("/api/login/<email>")
def api_login(email):
state = CAaaState()
user_id = state.get_user_id(email)
if user_id is None:
user_id = state.new_user(email)
return jsonify(user_id=user_id)
@app.route("/api/status") @app.route("/api/status")
def api_status(): def api_status():
if time.time() - sm.last_update_timestamp > STATS_CACHING_EXPIRATION: if time.time() - sm.last_update_timestamp > STATS_CACHING_EXPIRATION:
......
...@@ -33,23 +33,10 @@ class CAaaState: ...@@ -33,23 +33,10 @@ class CAaaState:
self.cnx.commit() self.cnx.commit()
cursor.close() cursor.close()
def _check_user(self, username): def new_user(self, email):
cursor = self._get_cursor(dictionary=True)
q = "SELECT id FROM users WHERE email=%s"
cursor.execute(q, (username,))
if cursor.rowcount == 0:
self._close_cursor(cursor)
return self._create_user(username)
else:
row = cursor.fetchone()
self._close_cursor(cursor)
return row["id"]
def _create_user(self, username):
cursor = self._get_cursor() cursor = self._get_cursor()
q = "INSERT INTO users (email) VALUES (%s)" q = "INSERT INTO users (email) VALUES (%s)"
cursor.execute(q, (username,)) cursor.execute(q, (email,))
user_id = cursor.lastrowid user_id = cursor.lastrowid
self._close_cursor(cursor) self._close_cursor(cursor)
return user_id return user_id
...@@ -62,8 +49,16 @@ class CAaaState: ...@@ -62,8 +49,16 @@ class CAaaState:
self._close_cursor(cursor) self._close_cursor(cursor)
return count == 1 return count == 1
def get_user_id(self, username): def get_user_id(self, email):
return self._check_user(username) cursor = self._get_cursor()
q = "SELECT id FROM users WHERE email=%s"
cursor.execute(q, (email,))
if cursor.rowcount == 0:
return None
else:
row = cursor.fetchone()[0]
self._close_cursor(cursor)
return row
def get_user_email(self, user_id): def get_user_email(self, user_id):
cursor = self._get_cursor() cursor = self._get_cursor()
......
{% extends "base_user.html" %} {% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %} {% block title %}User usage history{% endblock %}
{% block content %} {% block content %}
<h2>Status overview for user {{ user }}</h2> <h2>Usage history</h2>
{% if apps|length == 0 and not has_notebook %} {% if apps|length == 0 and not has_notebook %}
<p>You have no clusters running at this time</p> <p>You have no clusters running at this time</p>
{% endif %} {% endif %}
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
{% if has_notebook %} {% if has_notebook %}
<p>You have a notebook running: <p>You have a notebook running:
<a href="{{ notebook_address }}">open</a> <a href="{{ notebook_address }}">open</a>
<a href="{{ url_for("web_terminate", username=user, cluster_id=notebook_cluster_id) }}">terminate</a> <a href="{{ url_for("web_terminate", user_id=user_id, cluster_id=notebook_cluster_id) }}">terminate</a>
</p> </p>
{% endif %} {% endif %}
...@@ -35,15 +35,15 @@ ...@@ -35,15 +35,15 @@
<td>{{ a["status"] }}</td> <td>{{ a["status"] }}</td>
<td> <td>
{% if a["status"] == "finished" %} {% if a["status"] == "finished" %}
<a href="{{ url_for("api_history_log_archive", username=user, app_id=a["id"]) }}">Zip</a> <a href="{{ url_for("api_history_log_archive", user_id=user_id, app_id=a["id"]) }}">Zip</a>
{% endif %} {% endif %}
{% if a["status"] == "running" %} {% if a["status"] == "running" %}
<a href="{{ url_for("web_inspect", username=user, app_id=a["cluster_id"]) }}">Inspect</a> <a href="{{ url_for("web_inspect", user_id=user_id, app_id=a["cluster_id"]) }}">Inspect</a>
{% endif %} {% endif %}
</td> </td>
<td> <td>
{% if a["status"] == "running" %} {% if a["status"] == "running" %}
{{ url_for("web_terminate", username=user, cluster_id=a["cluster_id"]) }} {{ url_for("web_terminate", user_id=user_id, cluster_id=a["cluster_id"]) }}
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
......
{% extends "base.html" %} {% extends "base.html" %}
{% block footer %} {% block footer %}
<p>Back to the <a href="/web/{{ user }}">user home page</a></p> <p>Back to the <a href="/web/{{ user_id }}">user home page</a></p>
{{ super() }} {{ super() }}
{% endblock %} {% endblock %}
...@@ -3,13 +3,13 @@ ...@@ -3,13 +3,13 @@
{% block content %} {% block content %}
<p>Start an activity:</p> <p>Start an activity:</p>
<ul> <ul>
<li><a href="{{ url_for("web_notebook", username=user) }}">Give me a Spark notebook</a></li> <li><a href="{{ url_for("web_notebook", user_id=user_id) }}">Give me a Spark notebook</a></li>
<li><a href="{{ url_for("web_spark_submit", username=user) }}">Submit a Spark application</a></li> <li><a href="{{ url_for("web_spark_submit", user_id=user_id) }}">Submit a Spark application</a></li>
</ul> </ul>
<p>My activities:</p> <p>My activities:</p>
<ul> <ul>
<li><a href="{{ url_for("web_user_apps", username=user) }}">Applications</a></li> <li><a href="{{ url_for("web_user_apps", user_id=user_id) }}">Applications</a></li>
</ul> </ul>
<p>Cluster status:</p> <p>Cluster status:</p>
...@@ -17,6 +17,6 @@ ...@@ -17,6 +17,6 @@
<li><a href="{{ url_for("web_status") }}">Overview</a></li> <li><a href="{{ url_for("web_status") }}">Overview</a></li>
</ul> </ul>
<p>Your email address used for notifications is: {{ email }}</p>
{% endblock %} {% endblock %}
\ No newline at end of file
...@@ -5,15 +5,14 @@ ...@@ -5,15 +5,14 @@
<script> <script>
function process() function process()
{ {
var url = "http://" + window.location.hostname + "/web/" + $("#user").val(); location.href = "http://" + window.location.hostname + "/web/login/" + $("#user").val();
location.href = url;
return false; return false;
} }
</script> </script>
<form onSubmit="return process();"> <form onSubmit="return process();">
<label for="user">Type your username to start:</label> <label for="user">Type your username to start:</label>
<input id="user" autofocus required type="email" autocomplete="on"> <input id="user" autofocus required type="email" autocomplete="on" name="email">
<input type="submit" value="Start"> <input type="submit" value="Start">
</form> </form>
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
{% for p in c[1] %} {% for p in c[1] %}
<li><a href="{{ p[1] }}">{{ p[0] }}</a></li> <li><a href="{{ p[1] }}">{{ p[0] }}</a></li>
{% endfor %} {% endfor %}
<li><a href="/web/{{ user }}/container/{{ c[2] }}/logs">Logs</a></li> <li><a href="{{ url_for("web_logs", user_id=user_id, container_id=c[2]) }}">Logs</a></li>
</ul> </ul>
{% endfor %} {% endfor %}
</ul> </ul>
......
...@@ -9,11 +9,10 @@ ...@@ -9,11 +9,10 @@
<li>Memory limit: {{ MemoryLimit }}</li> <li>Memory limit: {{ MemoryLimit }}</li>
<li>NCPU: {{ NCPU }}</li> <li>NCPU: {{ NCPU }}</li>
<li>Driver status:</li> <li>Driver status:</li>
<ul> <li><ul>
{% for entry in DriverStatus %} {% for entry in DriverStatus %}
<li>{{ entry[0] }}: {{ entry[1] }}</li> <li>{{ entry[0] }}: {{ entry[1] }}</li>
{% endfor %} {% endfor %}
</ul></li>
</ul> </ul>
</ul>
<div id="userlist"></div>
{% endblock %} {% endblock %}
\ No newline at end of file
...@@ -2,4 +2,4 @@ from uuid import uuid1 as uuid ...@@ -2,4 +2,4 @@ from uuid import uuid1 as uuid
def get_uuid(): def get_uuid():
return str(uuid()) return str(uuid())
\ No newline at end of file
from flask import render_template from flask import render_template, redirect, url_for, abort
from caaas import app from caaas import app
from caaas.config_parser import config from caaas.config_parser import config
...@@ -18,23 +18,36 @@ def web_status(): ...@@ -18,23 +18,36 @@ def web_status():
return render_template('status.html', **status) return render_template('status.html', **status)
@app.route("/web/<username>") @app.route("/web/login/<email>")
def web_index(username): def web_login(email):
state = CAaaState() state = CAaaState()
state.get_user_id(username) # creates the user if it does not exists user_id = state.get_user_id(email)
if user_id is None:
user_id = state.new_user(email)
return redirect(url_for("web_index", user_id=user_id))
@app.route("/web/<user_id>")
def web_index(user_id):
state = CAaaState()
if not state.check_user_id(user_id):
return redirect(url_for('index'))
template_vars = { template_vars = {
"user": username "user_id": user_id,
"email": state.get_user_email(user_id)
} }
return render_template('home.html', **template_vars) return render_template('home.html', **template_vars)
@app.route("/web/<username>/apps") @app.route("/web/<user_id>/apps")
def web_user_apps(username): def web_user_apps(user_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
return redirect(url_for('index'))
apps = state.get_applications(user_id) apps = state.get_applications(user_id)
template_vars = { template_vars = {
"user": username, "user_id": user_id,
"apps": apps, "apps": apps,
"has_notebook": state.has_notebook(user_id), "has_notebook": state.has_notebook(user_id),
"notebook_address": sm.get_notebook(user_id), "notebook_address": sm.get_notebook(user_id),
...@@ -43,12 +56,14 @@ def web_user_apps(username): ...@@ -43,12 +56,14 @@ def web_user_apps(username):
return render_template('apps.html', **template_vars) return render_template('apps.html', **template_vars)
@app.route("/web/<username>/spark-notebook") @app.route("/web/<user_id>/spark-notebook")
def web_notebook(username): def web_notebook(user_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
return redirect(url_for('index'))
template_vars = { template_vars = {
"user": username, "user_id": user_id,
"notebook_address": sm.get_notebook(user_id), "notebook_address": sm.get_notebook(user_id),
"max_age": config.cleanup_notebooks_older_than, "max_age": config.cleanup_notebooks_older_than,
"wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning) "wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning)
...@@ -56,13 +71,15 @@ def web_notebook(username): ...@@ -56,13 +71,15 @@ def web_notebook(username):
return render_template('notebook.html', **template_vars) return render_template('notebook.html', **template_vars)
@app.route("/web/<username>/cluster/<cluster_id>/inspect") @app.route("/web/<user_id>/cluster/<cluster_id>/inspect")
def web_inspect(username, cluster_id): def web_inspect(user_id, cluster_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id) cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id: if cluster["user_id"] != user_id:
return "" # TODO abort(404)
containers = state.get_containers(cluster_id=cluster_id) containers = state.get_containers(cluster_id=cluster_id)
clist = [] clist = []
for cid, cinfo in containers.items(): for cid, cinfo in containers.items():
...@@ -71,55 +88,58 @@ def web_inspect(username, cluster_id): ...@@ -71,55 +88,58 @@ def web_inspect(username, cluster_id):
template_vars = { template_vars = {
"cluster_name": cluster["name"], "cluster_name": cluster["name"],
"containers": clist, "containers": clist,
"user": username "user_id": user_id
} }
return render_template('inspect.html', **template_vars) return render_template('inspect.html', **template_vars)
@app.route("/web/<username>/cluster/<cluster_id>/terminate") @app.route("/web/<user_id>/cluster/<cluster_id>/terminate")
def web_terminate(username, cluster_id): def web_terminate(user_id, cluster_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id) cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id: if cluster["user_id"] != user_id:
return "" # TODO abort(404)
template_vars = { template_vars = {
"cluster_name": cluster["name"], "cluster_name": cluster["name"],
"cluster_id": cluster_id, "cluster_id": cluster_id,
"user": username,
"user_id": user_id "user_id": user_id
} }
return render_template('terminate.html', **template_vars) return render_template('terminate.html', **template_vars)
@app.route("/web/<username>/container/<container_id>/logs") @app.route("/web/<user_id>/container/<container_id>/logs")
def web_logs(username, container_id): def web_logs(user_id, container_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
# FIXME: check user_id return redirect(url_for('index'))
cont = state.get_container(container_id) cont = state.get_container(container_id)
if user_id != cont["user_id"]:
abort(404)
logs = sm.get_log(container_id) logs = sm.get_log(container_id)
if logs is None: if logs is None:
ret = { abort(404)
'user': username
}
else: else:
logs = logs.decode("ascii") logs = logs.decode("ascii")
ret = { ret = {
'user': username, 'user_id': user_id,
'cont_contents': cont['contents'], 'cont_contents': cont['contents'],
"cont_logs": logs "cont_logs": logs
} }
return render_template('logs.html', **ret) return render_template('logs.html', **ret)
@app.route("/web/<username>/submit-spark-app") @app.route("/web/<user_id>/submit-spark-app")
def web_spark_submit(username): def web_spark_submit(user_id):
state = CAaaState() state = CAaaState()
user_id = state.get_user_id(username) if not state.check_user_id(user_id):
# FIXME: check user_id return redirect(url_for('index'))
template_vars = { template_vars = {
'user_id': user_id, 'user_id': user_id,
'user': username
} }
return render_template('submit.html', **template_vars) return render_template('submit.html', **template_vars)
...@@ -3,12 +3,12 @@ ...@@ -3,12 +3,12 @@
SPARK_VER=1.4.1 SPARK_VER=1.4.1
HADOOP_VER=hadoop2.4 HADOOP_VER=hadoop2.4
python ./gen_dockerfiles.py $SPARK_VER $HADOOP_VER python ./gen_dockerfiles.py ${SPARK_VER} ${HADOOP_VER}
for d in master worker shell submit notebook; do for d in master worker shell submit notebook; do
cd $d cd $d
docker build -t 10.0.0.2:5000/venza/spark-$d:$SPARK_VER . docker build -t 10.0.0.2:5000/venza/spark-$d:${SPARK_VER} .
docker push 10.0.0.2:5000/venza/spark-$d:$SPARK_VER docker push 10.0.0.2:5000/venza/spark-$d:${SPARK_VER}
cd .. cd ..
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:$SPARK_VER docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:${SPARK_VER}
done done
...@@ -3,5 +3,5 @@ cd /opt/spark ...@@ -3,5 +3,5 @@ cd /opt/spark
export SPARK_LOCAL_IP=`awk 'NR==1 {print $1}' /etc/hosts` export SPARK_LOCAL_IP=`awk 'NR==1 {print $1}' /etc/hosts`
./bin/spark-class org.apache.spark.deploy.worker.Worker \ ./bin/spark-class org.apache.spark.deploy.worker.Worker \
spark://${SPARK_MASTER_IP}:7077 --cores ${SPARK_WORKER_CORES:-4} --memory ${SPARK_WORKER_RAM:-4g} \ spark://${SPARK_MASTER_IP}:7077 --cores ${SPARK_WORKER_CORES:-4} --memory ${SPARK_WORKER_RAM:-4g} \
-h $SPARK_LOCAL_IP \ -h ${SPARK_LOCAL_IP} \
"$@" "$@"
...@@ -5,12 +5,13 @@ import shutil ...@@ -5,12 +5,13 @@ import shutil
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
def copyfile(fname, dest): def copyfile(fname, dest):
shutil.copy("files/" + fname, dest + "/files/" + fname) shutil.copy("files/" + fname, dest + "/files/" + fname)
if len(sys.argv) < 3: if len(sys.argv) < 3:
print("Usage: {} <spark version> <hadoop version>".format(sys.argv[0])) print("Usage: {} <spark version> <hadoop version>".format(sys.argv[0]))
sys.exit(1) sys.exit(1)
ji_env = Environment(loader=FileSystemLoader('templates')) ji_env = Environment(loader=FileSystemLoader('templates'))
...@@ -22,31 +23,31 @@ master_tmpl = ji_env.get_template('master.tmpl') ...@@ -22,31 +23,31 @@ master_tmpl = ji_env.get_template('master.tmpl')
master = common + "\n" + master_tmpl.render() master = common + "\n" + master_tmpl.render()
open("master/Dockerfile", "w").write(master) open("master/Dockerfile", "w").write(master)
for f in ["remove_alias.sh", "start-master.sh"]: for f in ["remove_alias.sh", "start-master.sh"]:
copyfile(f, "master") copyfile(f, "master")
# Worker # Worker
worker_tmpl = ji_env.get_template('worker.tmpl') worker_tmpl = ji_env.get_template('worker.tmpl')
worker = common + "\n" + worker_tmpl.render() worker = common + "\n" + worker_tmpl.render()
open("worker/Dockerfile", "w").write(worker) open("worker/Dockerfile", "w").write(worker)
for f in ["remove_alias.sh", "start-worker.sh"]: for f in ["remove_alias.sh", "start-worker.sh"]:
copyfile(f, "worker") copyfile(f, "worker")
# Shell # Shell
shell_tmpl = ji_env.get_template('shell.tmpl') shell_tmpl = ji_env.get_template('shell.tmpl')
shell = common + "\n" + shell_tmpl.render() shell = common + "\n" + shell_tmpl.render()
open("shell/Dockerfile", "w").write(shell) open("shell/Dockerfile", "w").write(shell)
for f in ["remove_alias.sh", "start-shell.sh"]: for f in ["remove_alias.sh", "start-shell.sh"]:
copyfile(f, "shell") copyfile(f, "shell")
# Submit # Submit
submit_tmpl = ji_env.get_template('submit.tmpl') submit_tmpl = ji_env.get_template('submit.tmpl')
submit = common + "\n" + submit_tmpl.render() submit = common + "\n" + submit_tmpl.render()
open("submit/Dockerfile", "w").write(submit) open("submit/Dockerfile", "w").write(submit)
for f in ["remove_alias.sh", "submit.sh"]: for f in ["remove_alias.sh", "submit.sh"]:
copyfile(f, "submit") copyfile(f, "submit")
# Notebook # Notebook
#nb_tmpl = ji_env.get_template('notebook.tmpl') # nb_tmpl = ji_env.get_template('notebook.tmpl')
#nb = nb_tmpl.render() # nb = nb_tmpl.render()
#open("notebook/Dockerfile", "w").write(nb) # open("notebook/Dockerfile", "w").write(nb)
...@@ -6,6 +6,4 @@ export SPARK_MASTER_WEBUI_PORT=8080 ...@@ -6,6 +6,4 @@ export SPARK_MASTER_WEBUI_PORT=8080
cd /opt/spark cd /opt/spark
./bin/spark-class org.apache.spark.deploy.master.Master \ ./bin/spark-class org.apache.spark.deploy.master.Master \
--host $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \ --host ${SPARK_MASTER_IP} --port ${SPARK_MASTER_PORT} --webui-port ${SPARK_MASTER_WEBUI_PORT} $@
$@
...@@ -3,5 +3,5 @@ ...@@ -3,5 +3,5 @@
SPARK_VER=1.4.1 SPARK_VER=1.4.1
for d in master worker shell submit notebook; do for d in master worker shell submit notebook; do
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:$SPARK_VER docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-${d}:${SPARK_VER}
done done
...@@ -3,5 +3,4 @@ cd /opt/spark ...@@ -3,5 +3,4 @@ cd /opt/spark
export SPARK_LOCAL_IP=`awk 'NR==1 {print $1}' /etc/hosts`