Commit ca9a4ef9 authored by Daniele Venzano's avatar Daniele Venzano

Use user_id also in web urls

parent 899db8e5
This diff is collapsed.
......@@ -3,5 +3,3 @@ app = Flask(__name__)
import caaas.web
import caaas.api
......@@ -10,6 +10,15 @@ from caaas.swarm_manager import sm
STATS_CACHING_EXPIRATION = 1 # seconds
@app.route("/api/login/<email>")
def api_login(email):
state = CAaaState()
user_id = state.get_user_id(email)
if user_id is None:
user_id = state.new_user(email)
return jsonify(user_id=user_id)
@app.route("/api/status")
def api_status():
if time.time() - sm.last_update_timestamp > STATS_CACHING_EXPIRATION:
......
......@@ -33,23 +33,10 @@ class CAaaState:
self.cnx.commit()
cursor.close()
def _check_user(self, username):
cursor = self._get_cursor(dictionary=True)
q = "SELECT id FROM users WHERE email=%s"
cursor.execute(q, (username,))
if cursor.rowcount == 0:
self._close_cursor(cursor)
return self._create_user(username)
else:
row = cursor.fetchone()
self._close_cursor(cursor)
return row["id"]
def _create_user(self, username):
def new_user(self, email):
cursor = self._get_cursor()
q = "INSERT INTO users (email) VALUES (%s)"
cursor.execute(q, (username,))
cursor.execute(q, (email,))
user_id = cursor.lastrowid
self._close_cursor(cursor)
return user_id
......@@ -62,8 +49,16 @@ class CAaaState:
self._close_cursor(cursor)
return count == 1
def get_user_id(self, username):
return self._check_user(username)
def get_user_id(self, email):
cursor = self._get_cursor()
q = "SELECT id FROM users WHERE email=%s"
cursor.execute(q, (email,))
if cursor.rowcount == 0:
return None
else:
row = cursor.fetchone()[0]
self._close_cursor(cursor)
return row
def get_user_email(self, user_id):
cursor = self._get_cursor()
......
{% extends "base_user.html" %}
{% block title %}{{ user }} status{% endblock %}
{% block title %}User usage history{% endblock %}
{% block content %}
<h2>Status overview for user {{ user }}</h2>
<h2>Usage history</h2>
{% if apps|length == 0 and not has_notebook %}
<p>You have no clusters running at this time</p>
{% endif %}
......@@ -9,7 +9,7 @@
{% if has_notebook %}
<p>You have a notebook running:
<a href="{{ notebook_address }}">open</a>
<a href="{{ url_for("web_terminate", username=user, cluster_id=notebook_cluster_id) }}">terminate</a>
<a href="{{ url_for("web_terminate", user_id=user_id, cluster_id=notebook_cluster_id) }}">terminate</a>
</p>
{% endif %}
......@@ -35,15 +35,15 @@
<td>{{ a["status"] }}</td>
<td>
{% if a["status"] == "finished" %}
<a href="{{ url_for("api_history_log_archive", username=user, app_id=a["id"]) }}">Zip</a>
<a href="{{ url_for("api_history_log_archive", user_id=user_id, app_id=a["id"]) }}">Zip</a>
{% endif %}
{% if a["status"] == "running" %}
<a href="{{ url_for("web_inspect", username=user, app_id=a["cluster_id"]) }}">Inspect</a>
<a href="{{ url_for("web_inspect", user_id=user_id, app_id=a["cluster_id"]) }}">Inspect</a>
{% endif %}
</td>
<td>
{% if a["status"] == "running" %}
{{ url_for("web_terminate", username=user, cluster_id=a["cluster_id"]) }}
{{ url_for("web_terminate", user_id=user_id, cluster_id=a["cluster_id"]) }}
{% endif %}
</td>
</tr>
......
{% extends "base.html" %}
{% block footer %}
<p>Back to the <a href="/web/{{ user }}">user home page</a></p>
<p>Back to the <a href="/web/{{ user_id }}">user home page</a></p>
{{ super() }}
{% endblock %}
......@@ -3,13 +3,13 @@
{% block content %}
<p>Start an activity:</p>
<ul>
<li><a href="{{ url_for("web_notebook", username=user) }}">Give me a Spark notebook</a></li>
<li><a href="{{ url_for("web_spark_submit", username=user) }}">Submit a Spark application</a></li>
<li><a href="{{ url_for("web_notebook", user_id=user_id) }}">Give me a Spark notebook</a></li>
<li><a href="{{ url_for("web_spark_submit", user_id=user_id) }}">Submit a Spark application</a></li>
</ul>
<p>My activities:</p>
<ul>
<li><a href="{{ url_for("web_user_apps", username=user) }}">Applications</a></li>
<li><a href="{{ url_for("web_user_apps", user_id=user_id) }}">Applications</a></li>
</ul>
<p>Cluster status:</p>
......@@ -17,6 +17,6 @@
<li><a href="{{ url_for("web_status") }}">Overview</a></li>
</ul>
<p>Your email address used for notifications is: {{ email }}</p>
{% endblock %}
\ No newline at end of file
......@@ -5,15 +5,14 @@
<script>
function process()
{
var url = "http://" + window.location.hostname + "/web/" + $("#user").val();
location.href = url;
location.href = "http://" + window.location.hostname + "/web/login/" + $("#user").val();
return false;
}
</script>
<form onSubmit="return process();">
<label for="user">Type your username to start:</label>
<input id="user" autofocus required type="email" autocomplete="on">
<input id="user" autofocus required type="email" autocomplete="on" name="email">
<input type="submit" value="Start">
</form>
......
......@@ -10,7 +10,7 @@
{% for p in c[1] %}
<li><a href="{{ p[1] }}">{{ p[0] }}</a></li>
{% endfor %}
<li><a href="/web/{{ user }}/container/{{ c[2] }}/logs">Logs</a></li>
<li><a href="{{ url_for("web_logs", user_id=user_id, container_id=c[2]) }}">Logs</a></li>
</ul>
{% endfor %}
</ul>
......
......@@ -9,11 +9,10 @@
<li>Memory limit: {{ MemoryLimit }}</li>
<li>NCPU: {{ NCPU }}</li>
<li>Driver status:</li>
<ul>
<li><ul>
{% for entry in DriverStatus %}
<li>{{ entry[0] }}: {{ entry[1] }}</li>
{% endfor %}
</ul></li>
</ul>
</ul>
<div id="userlist"></div>
{% endblock %}
\ No newline at end of file
......@@ -2,4 +2,4 @@ from uuid import uuid1 as uuid
def get_uuid():
return str(uuid())
\ No newline at end of file
return str(uuid())
from flask import render_template
from flask import render_template, redirect, url_for, abort
from caaas import app
from caaas.config_parser import config
......@@ -18,23 +18,36 @@ def web_status():
return render_template('status.html', **status)
@app.route("/web/<username>")
def web_index(username):
@app.route("/web/login/<email>")
def web_login(email):
state = CAaaState()
state.get_user_id(username) # creates the user if it does not exists
user_id = state.get_user_id(email)
if user_id is None:
user_id = state.new_user(email)
return redirect(url_for("web_index", user_id=user_id))
@app.route("/web/<user_id>")
def web_index(user_id):
state = CAaaState()
if not state.check_user_id(user_id):
return redirect(url_for('index'))
template_vars = {
"user": username
"user_id": user_id,
"email": state.get_user_email(user_id)
}
return render_template('home.html', **template_vars)
@app.route("/web/<username>/apps")
def web_user_apps(username):
@app.route("/web/<user_id>/apps")
def web_user_apps(user_id):
state = CAaaState()
user_id = state.get_user_id(username)
if not state.check_user_id(user_id):
return redirect(url_for('index'))
apps = state.get_applications(user_id)
template_vars = {
"user": username,
"user_id": user_id,
"apps": apps,
"has_notebook": state.has_notebook(user_id),
"notebook_address": sm.get_notebook(user_id),
......@@ -43,12 +56,14 @@ def web_user_apps(username):
return render_template('apps.html', **template_vars)
@app.route("/web/<username>/spark-notebook")
def web_notebook(username):
@app.route("/web/<user_id>/spark-notebook")
def web_notebook(user_id):
state = CAaaState()
user_id = state.get_user_id(username)
if not state.check_user_id(user_id):
return redirect(url_for('index'))
template_vars = {
"user": username,
"user_id": user_id,
"notebook_address": sm.get_notebook(user_id),
"max_age": config.cleanup_notebooks_older_than,
"wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning)
......@@ -56,13 +71,15 @@ def web_notebook(username):
return render_template('notebook.html', **template_vars)
@app.route("/web/<username>/cluster/<cluster_id>/inspect")
def web_inspect(username, cluster_id):
@app.route("/web/<user_id>/cluster/<cluster_id>/inspect")
def web_inspect(user_id, cluster_id):
state = CAaaState()
user_id = state.get_user_id(username)
if not state.check_user_id(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id:
return "" # TODO
abort(404)
containers = state.get_containers(cluster_id=cluster_id)
clist = []
for cid, cinfo in containers.items():
......@@ -71,55 +88,58 @@ def web_inspect(username, cluster_id):
template_vars = {
"cluster_name": cluster["name"],
"containers": clist,
"user": username
"user_id": user_id
}
return render_template('inspect.html', **template_vars)
@app.route("/web/<username>/cluster/<cluster_id>/terminate")
def web_terminate(username, cluster_id):
@app.route("/web/<user_id>/cluster/<cluster_id>/terminate")
def web_terminate(user_id, cluster_id):
state = CAaaState()
user_id = state.get_user_id(username)
if not state.check_user_id(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id:
return "" # TODO
abort(404)
template_vars = {
"cluster_name": cluster["name"],
"cluster_id": cluster_id,
"user": username,
"user_id": user_id
}
return render_template('terminate.html', **template_vars)
@app.route("/web/<username>/container/<container_id>/logs")
def web_logs(username, container_id):
@app.route("/web/<user_id>/container/<container_id>/logs")
def web_logs(user_id, container_id):
state = CAaaState()
user_id = state.get_user_id(username)
# FIXME: check user_id
if not state.check_user_id(user_id):
return redirect(url_for('index'))
cont = state.get_container(container_id)
if user_id != cont["user_id"]:
abort(404)
logs = sm.get_log(container_id)
if logs is None:
ret = {
'user': username
}
abort(404)
else:
logs = logs.decode("ascii")
ret = {
'user': username,
'user_id': user_id,
'cont_contents': cont['contents'],
"cont_logs": logs
}
return render_template('logs.html', **ret)
return render_template('logs.html', **ret)
@app.route("/web/<username>/submit-spark-app")
def web_spark_submit(username):
@app.route("/web/<user_id>/submit-spark-app")
def web_spark_submit(user_id):
state = CAaaState()
user_id = state.get_user_id(username)
# FIXME: check user_id
if not state.check_user_id(user_id):
return redirect(url_for('index'))
template_vars = {
'user_id': user_id,
'user': username
}
return render_template('submit.html', **template_vars)
......@@ -3,12 +3,12 @@
SPARK_VER=1.4.1
HADOOP_VER=hadoop2.4
python ./gen_dockerfiles.py $SPARK_VER $HADOOP_VER
python ./gen_dockerfiles.py ${SPARK_VER} ${HADOOP_VER}
for d in master worker shell submit notebook; do
cd $d
docker build -t 10.0.0.2:5000/venza/spark-$d:$SPARK_VER .
docker push 10.0.0.2:5000/venza/spark-$d:$SPARK_VER
docker build -t 10.0.0.2:5000/venza/spark-$d:${SPARK_VER} .
docker push 10.0.0.2:5000/venza/spark-$d:${SPARK_VER}
cd ..
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:$SPARK_VER
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:${SPARK_VER}
done
......@@ -3,5 +3,5 @@ cd /opt/spark
export SPARK_LOCAL_IP=`awk 'NR==1 {print $1}' /etc/hosts`
./bin/spark-class org.apache.spark.deploy.worker.Worker \
spark://${SPARK_MASTER_IP}:7077 --cores ${SPARK_WORKER_CORES:-4} --memory ${SPARK_WORKER_RAM:-4g} \
-h $SPARK_LOCAL_IP \
-h ${SPARK_LOCAL_IP} \
"$@"
......@@ -5,12 +5,13 @@ import shutil
from jinja2 import Environment, FileSystemLoader
def copyfile(fname, dest):
shutil.copy("files/" + fname, dest + "/files/" + fname)
shutil.copy("files/" + fname, dest + "/files/" + fname)
if len(sys.argv) < 3:
print("Usage: {} <spark version> <hadoop version>".format(sys.argv[0]))
sys.exit(1)
print("Usage: {} <spark version> <hadoop version>".format(sys.argv[0]))
sys.exit(1)
ji_env = Environment(loader=FileSystemLoader('templates'))
......@@ -22,31 +23,31 @@ master_tmpl = ji_env.get_template('master.tmpl')
master = common + "\n" + master_tmpl.render()
open("master/Dockerfile", "w").write(master)
for f in ["remove_alias.sh", "start-master.sh"]:
copyfile(f, "master")
copyfile(f, "master")
# Worker
worker_tmpl = ji_env.get_template('worker.tmpl')
worker = common + "\n" + worker_tmpl.render()
open("worker/Dockerfile", "w").write(worker)
for f in ["remove_alias.sh", "start-worker.sh"]:
copyfile(f, "worker")
copyfile(f, "worker")
# Shell
shell_tmpl = ji_env.get_template('shell.tmpl')
shell = common + "\n" + shell_tmpl.render()
open("shell/Dockerfile", "w").write(shell)
for f in ["remove_alias.sh", "start-shell.sh"]:
copyfile(f, "shell")
copyfile(f, "shell")
# Submit
submit_tmpl = ji_env.get_template('submit.tmpl')
submit = common + "\n" + submit_tmpl.render()
open("submit/Dockerfile", "w").write(submit)
for f in ["remove_alias.sh", "submit.sh"]:
copyfile(f, "submit")
copyfile(f, "submit")
# Notebook
#nb_tmpl = ji_env.get_template('notebook.tmpl')
#nb = nb_tmpl.render()
#open("notebook/Dockerfile", "w").write(nb)
# nb_tmpl = ji_env.get_template('notebook.tmpl')
# nb = nb_tmpl.render()
# open("notebook/Dockerfile", "w").write(nb)
......@@ -6,6 +6,4 @@ export SPARK_MASTER_WEBUI_PORT=8080
cd /opt/spark
./bin/spark-class org.apache.spark.deploy.master.Master \
--host $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \
$@
--host ${SPARK_MASTER_IP} --port ${SPARK_MASTER_PORT} --webui-port ${SPARK_MASTER_WEBUI_PORT} $@
......@@ -3,5 +3,5 @@
SPARK_VER=1.4.1
for d in master worker shell submit notebook; do
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-$d:$SPARK_VER
docker -H 10.0.0.2:2380 pull 10.0.0.2:5000/venza/spark-${d}:${SPARK_VER}
done
......@@ -3,5 +3,4 @@ cd /opt/spark
export SPARK_LOCAL_IP=`awk 'NR==1 {print $1}' /etc/hosts`
./bin/spark-class org.apache.spark.deploy.worker.Worker \
spark://${SPARK_MASTER_IP}:7077 --cores ${SPARK_WORKER_CORES:-4} --memory ${SPARK_WORKER_RAM:-4g} \
-h $SPARK_LOCAL_IP \
"$@"
-h ${SPARK_LOCAL_IP} "$@"
......@@ -3,7 +3,7 @@
-- http://www.phpmyadmin.net
--
-- Host: localhost
-- Generation Time: Aug 06, 2015 at 10:16 AM
-- Generation Time: Aug 13, 2015 at 03:56 PM
-- Server version: 5.5.44-MariaDB-1ubuntu0.14.04.1
-- PHP Version: 5.5.9-1ubuntu4.11
......@@ -24,6 +24,26 @@ USE `caaas`;
-- --------------------------------------------------------
--
-- Table structure for table `applications`
--
DROP TABLE IF EXISTS `applications`;
CREATE TABLE IF NOT EXISTS `applications` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`execution_name` varchar(512) NOT NULL,
`cmd` varchar(1024) NOT NULL,
`spark_options` varchar(1024) NOT NULL,
`user_id` int(11) NOT NULL,
`time_started` timestamp NULL DEFAULT NULL,
`time_finished` timestamp NULL DEFAULT NULL,
`cluster_id` int(11) DEFAULT NULL,
`status` varchar(16) CHARACTER SET ascii NOT NULL DEFAULT 'setup',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=22 ;
-- --------------------------------------------------------
--
-- Table structure for table `clusters`
--
......@@ -34,8 +54,9 @@ CREATE TABLE IF NOT EXISTS `clusters` (
`user_id` int(11) NOT NULL,
`master_address` varchar(512) CHARACTER SET ascii NOT NULL,
`name` varchar(256) NOT NULL,
`time_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=11 ;
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=53 ;
-- --------------------------------------------------------
......@@ -52,24 +73,7 @@ CREATE TABLE IF NOT EXISTS `containers` (
`ip_address` varchar(16) NOT NULL,
`contents` varchar(512) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=37 ;
-- --------------------------------------------------------
--
-- Table structure for table `notebooks`
--
DROP TABLE IF EXISTS `notebooks`;
CREATE TABLE IF NOT EXISTS `notebooks` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`cluster_id` int(11) NOT NULL,
`time_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`address` varchar(512) CHARACTER SET ascii NOT NULL,
`user_id` int(11) NOT NULL,
`container_id` int(11) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=10 ;
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=250 ;
-- --------------------------------------------------------
......@@ -79,14 +83,14 @@ CREATE TABLE IF NOT EXISTS `notebooks` (
DROP TABLE IF EXISTS `proxy`;
CREATE TABLE IF NOT EXISTS `proxy` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`proxy_id` varchar(512) NOT NULL,
`url` varchar(512) NOT NULL,
`id` varchar(128) CHARACTER SET ascii NOT NULL,
`internal_url` varchar(512) DEFAULT NULL,
`cluster_id` int(11) NOT NULL,
`proxy_type` varchar(64) NOT NULL,
`service_name` varchar(64) DEFAULT NULL,
`container_id` int(11) NOT NULL,
`last_access` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=37 ;
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
......@@ -97,7 +101,7 @@ CREATE TABLE IF NOT EXISTS `proxy` (
DROP TABLE IF EXISTS `users`;
CREATE TABLE IF NOT EXISTS `users` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`username` varchar(32) NOT NULL,
`email` varchar(128) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=3 ;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment