Commit bd21031d authored by Daniele Venzano's avatar Daniele Venzano

Is now able to create notebook clusters

parent 48c44819
...@@ -2,13 +2,13 @@ ...@@ -2,13 +2,13 @@
<module type="PYTHON_MODULE" version="4"> <module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager"> <component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" /> <content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 2.7.9 (E:\User Software\WinPython-64bit-2.7.9.3\python-2.7.9.amd64\python.exe)" jdkType="Python SDK" /> <orderEntry type="jdk" jdkName="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
<component name="TemplatesService"> <component name="TemplatesService">
<option name="TEMPLATE_FOLDERS"> <option name="TEMPLATE_FOLDERS">
<list> <list>
<option value="$MODULE_DIR$/images/templates" /> <option value="$MODULE_DIR$/caaas/templates" />
</list> </list>
</option> </option>
</component> </component>
......
...@@ -19,6 +19,8 @@ ...@@ -19,6 +19,8 @@
<column name="docker_id" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/> <column name="docker_id" sqlType="VARCHAR" precision="1024" scale="0" nullable="false" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/> <column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/> <column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="ip_address" sqlType="VARCHAR" precision="16" scale="0" nullable="false" jdbcType="12"/>
<column name="contents" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<primary-key name="PRIMARY" columns="id"/> <primary-key name="PRIMARY" columns="id"/>
</table> </table>
<table name="notebooks" schema="" catalog="caaas" type="TABLE"> <table name="notebooks" schema="" catalog="caaas" type="TABLE">
...@@ -27,6 +29,16 @@ ...@@ -27,6 +29,16 @@
<column name="time_created" sqlType="TIMESTAMP" precision="19" scale="0" nullable="false" jdbcType="93" def="Q1VSUkVOVF9USU1FU1RBTVA="/> <column name="time_created" sqlType="TIMESTAMP" precision="19" scale="0" nullable="false" jdbcType="93" def="Q1VSUkVOVF9USU1FU1RBTVA="/>
<column name="address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/> <column name="address" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/> <column name="user_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="container_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<primary-key name="PRIMARY" columns="id"/>
</table>
<table name="proxy" schema="" catalog="caaas" type="TABLE">
<column name="id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4" autoIncrement="true"/>
<column name="proxy_id" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="url" sqlType="VARCHAR" precision="512" scale="0" nullable="false" jdbcType="12"/>
<column name="cluster_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<column name="proxy_type" sqlType="VARCHAR" precision="64" scale="0" nullable="false" jdbcType="12"/>
<column name="container_id" sqlType="INT" precision="10" scale="0" nullable="false" jdbcType="4"/>
<primary-key name="PRIMARY" columns="id"/> <primary-key name="PRIMARY" columns="id"/>
</table> </table>
<table name="users" schema="" catalog="caaas" type="TABLE"> <table name="users" schema="" catalog="caaas" type="TABLE">
......
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="DataSourceManagerImpl" format="xml" hash="3541978474"> <component name="DataSourceManagerImpl" format="xml" hash="2263423227">
<data-source source="LOCAL" name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0"> <data-source source="LOCAL" name="MySQL - @m1" uuid="a32fd6de-3ffa-40c0-9ec8-8953a89c53e0">
<driver-ref>mysql</driver-ref> <driver-ref>mysql</driver-ref>
<synchronize>true</synchronize> <synchronize>true</synchronize>
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="vm" deleteMissingItems="true" traceLevel="DETAILS">
<serverData>
<paths name="bfm2">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="vm">
<serverdata>
<mappings>
<mapping deploy="/caaas" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>
\ No newline at end of file
<component name="ProjectDictionaryState">
<dictionary name="venzano">
<words>
<w>caaas</w>
<w>jinja</w>
</words>
</dictionary>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/images/notebook/files/start-notebook.sh" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
...@@ -10,5 +10,5 @@ ...@@ -10,5 +10,5 @@
<ConfirmationsSetting value="0" id="Add" /> <ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" /> <ConfirmationsSetting value="0" id="Remove" />
</component> </component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.9 (E:\User Software\WinPython-64bit-2.7.9.3\python-2.7.9.amd64\python.exe)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.4.0 (sftp://ubuntu@192.168.45.25:22/usr/bin/python3)" project-jdk-type="Python SDK" />
</project> </project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="WebResourcesPaths">
<contentEntries>
<entry url="file://$PROJECT_DIR$">
<entryData>
<resourceRoots>
<path value="file://$PROJECT_DIR$/caaas/static" />
</resourceRoots>
</entryData>
</entry>
</contentEntries>
</component>
</project>
\ No newline at end of file
This diff is collapsed.
# CAaaS - Container Analytics as a Service
This web application uses a Docker Swarm cluster to run on-demand Spark clusters.
## Requirements
* MySQL to keep all the state
* Docker Swarm
* A Docker registry containing Spark images
* Apache to act as a reverse proxy
## Apache configuration
Put this in your virtual host entry:
```
ProxyHTMLLinks a href
ProxyHTMLLinks area href
ProxyHTMLLinks link href
ProxyHTMLLinks img src longdesc usemap
ProxyHTMLLinks object classid codebase data usemap
ProxyHTMLLinks q cite
ProxyHTMLLinks blockquote cite
ProxyHTMLLinks ins cite
ProxyHTMLLinks del cite
ProxyHTMLLinks form action
ProxyHTMLLinks input src usemap
ProxyHTMLLinks head profile
ProxyHTMLLinks base href
ProxyHTMLLinks script src for
ProxyHTMLEvents onclick ondblclick onmousedown onmouseup \
onmouseover onmousemove onmouseout onkeypress \
onkeydown onkeyup onfocus onblur onload \
onunload onsubmit onreset onselect onchange
ProxyRequests Off
IncludeOptional /tmp/caaas-proxy.conf*
```
If you need to proxy the web application itself, add also these directives:
```
<Location /web/>
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass http://192.168.45.25:5000/web/
ProxyPassReverse http://192.168.45.25:5000/web/
ProxyHTMLURLMap /web/ /web/
</Location>
<Location /api/>
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass http://192.168.45.25:5000/api/
ProxyPassReverse http://192.168.45.25:5000/api/
ProxyHTMLURLMap /api/ /api/
</Location>
```
The script `apache-proxy.py` needs to be run to update apache whenever containers are created or destroyed.
\ No newline at end of file
from configparser import ConfigParser
from jinja2 import Template
from hashlib import md5
from os import system
from time import sleep
from urllib.parse import urlparse
from caaas import init_db, get_db
OUTFILE = "/tmp/caaas-proxy.conf"
ENTRY_TEMPLATE = """
<Location /proxy/{{ proxy_id }}>
ProxyHtmlEnable On
ProxyHTMLExtended On
ProxyPass {{ proxy_url }}
ProxyPassReverse {{ proxy_url }}
{% if proxy_type != "spark-notebook" %}
ProxyHTMLURLMap ^/(.*)$ /proxy/{{ proxy_id }}/$1 RL
{% endif %}
</Location>
{% if proxy_type == "spark-notebook" %}
<Location /proxy/{{ proxy_id }}/ws/>
ProxyPass ws://{{ netloc }}/proxy/{{ proxy_id }}/ws/
</Location>
{% endif %}
"""
def read_config():
conf = ConfigParser()
conf.read('caaas.ini')
return conf
def get_proxy_entries():
db = get_db()
return db.get_all_proxy()
def generate_file(proxy_entries):
output = ""
jinja_template = Template(ENTRY_TEMPLATE)
for proxy_id, proxy_url, proxy_type in proxy_entries:
netloc = urlparse(proxy_url)["netloc"]
jinja_dict = {
"proxy_id": proxy_id,
"proxy_url": proxy_url,
"proxy_type": proxy_type,
"netloc": netloc
}
apache_entry = jinja_template.render(jinja_dict)
output += apache_entry + "\n"
return output
def check_difference(generated_file):
m_new = md5()
m_new.update(generated_file.encode('ascii'))
m_old = md5()
try:
m_old.update(open(OUTFILE).read().encode('ascii'))
except FileNotFoundError:
return True
return m_new.digest() != m_old.digest()
def commit_and_reload(generated_file):
print("Apache config requires an update, committing and reloading")
open(OUTFILE, "w").write(generated_file)
system("sudo service apache2 reload")
if __name__ == "__main__":
conf = read_config()
init_db(conf['db']['user'], conf['db']['pass'], conf['db']['server'], conf['db']['db'])
print("CAaaS Apache proxy synchronizer starting")
while True:
print("Looping...")
entries = get_proxy_entries()
output = generate_file(entries)
if check_difference(output):
commit_and_reload(output)
sleep(1)
from caaas.sql import init_db from caaas.sql import CAaaSSQL
from caaas.swarm import swarm
from flask import Flask from flask import Flask
app = Flask(__name__) app = Flask(__name__)
_db = None
def init_db(user, passw, server, dbname):
global _db
_db = CAaaSSQL()
_db.connect(user, passw, server, dbname)
def get_db():
"""
:rtype: CAaaSSQL
"""
return _db
from caaas.swarm import swarm
import caaas.web import caaas.web
import caaas.api import caaas.api
from flask import jsonify from flask import jsonify
from caaas import app from caaas import app, swarm, get_db
from caaas.swarm import swarm
@app.route("/api/status") @app.route("/api/status")
def api_status(): def api_status():
template_variables = { data = {
'num_containers': int(swarm.status.num_containers), 'num_containers': int(swarm.status.num_containers),
'num_nodes': int(swarm.status.num_nodes) 'num_nodes': int(swarm.status.num_nodes)
} }
return jsonify(**template_variables) return jsonify(**data)
@app.route("/api/full-status")
def api_full_status():
db = get_db()
data = {}
user_list = db.get_all_users()
for user_id, username in user_list:
data[username] = {}
data[username]["num_clusters"] = db.count_clusters(user_id)
data[username]["num_containers"] = db.count_containers(user_id)
data[username]["has_notebook"] = db.has_notebook(user_id)
return jsonify(**data)
import mysql.connector import mysql.connector
from hashlib import md5
db = None
class CAaaSSQL: class CAaaSSQL:
...@@ -9,7 +8,7 @@ class CAaaSSQL: ...@@ -9,7 +8,7 @@ class CAaaSSQL:
def connect(self, user, passw, server, dbname): def connect(self, user, passw, server, dbname):
assert self.cnx is None assert self.cnx is None
self.cnx = mysql.connector.connect(user=user, password=passw, host=server, database=dbname) self.cnx = mysql.connector.connect(user=user, password=passw, host=server, database=dbname, buffered=True)
def _check_user(self, username): def _check_user(self, username):
cursor = self.cnx.cursor(dictionary=True) cursor = self.cnx.cursor(dictionary=True)
...@@ -27,14 +26,127 @@ class CAaaSSQL: ...@@ -27,14 +26,127 @@ class CAaaSSQL:
def _create_user(self, username): def _create_user(self, username):
cursor = self.cnx.cursor() cursor = self.cnx.cursor()
q = "INSERT INTO users (username) VALUES (%s)" q = "INSERT INTO users (username) VALUES (%s)"
cursor.execute(q, username) cursor.execute(q, (username,))
user_id = cursor.lastrowid user_id = cursor.lastrowid
self.cnx.commit() self.cnx.commit()
cursor.close() cursor.close()
return user_id return user_id
def get_user_id(self, username):
return self._check_user(username)
def get_all_users(self):
cursor = self.cnx.cursor()
q = "SELECT id, username FROM users"
user_list = []
cursor.execute(q)
for row in cursor:
user_list.append(row)
cursor.close()
return user_list
def count_clusters(self, user_id=None):
cursor = self.cnx.cursor()
if user_id is None:
q = "SELECT COUNT(*) FROM clusters"
cursor.execute(q)
else:
q = "SELECT COUNT(*) FROM clusters WHERE user_id=%s"
cursor.execute(q, (user_id,))
row = cursor.fetchone()
cursor.close()
return row[0]
def count_containers(self, user_id=None):
cursor = self.cnx.cursor()
if user_id is None:
q = "SELECT COUNT(*) FROM containers"
cursor.execute(q)
else:
q = "SELECT COUNT(*) FROM containers WHERE user_id=%s"
cursor.execute(q, (user_id,))
row = cursor.fetchone()
cursor.close()
return row[0]
def get_notebook(self, user_id):
cursor = self.cnx.cursor(dictionary=True)
q = "SELECT * FROM notebooks WHERE user_id=%s"
cursor.execute(q, (user_id,))
if cursor.rowcount == 0:
cursor.close()
return None
else:
row = cursor.fetchone()
cursor.close()
return row
def has_notebook(self, user_id):
ret = self.get_notebook(user_id)
return ret is not None
def init_db(user, passw, server, dbname): def get_url_proxy(self, proxy_id):
global db cursor = self.cnx.cursor()
db = CAaaSSQL() q = "SELECT url FROM proxy WHERE proxy_id=%s"
db.connect(user, passw, server, dbname) cursor.execute(q, (proxy_id,))
if cursor.rowcount == 0:
cursor.close()
return None
else:
row = cursor.fetchone()
cursor.close()
return row[0]
def get_all_proxy(self):
cursor = self.cnx.cursor()
q = "SELECT proxy_id, url, proxy_type, container_id FROM proxy"
cursor.execute(q)
proxy_list = []
for proxy_id, url, proxy_type, container_id in cursor:
proxy_list.append((proxy_id, url, proxy_type, container_id))
cursor.close()
return proxy_list
def new_cluster(self, user_id, name):
cursor = self.cnx.cursor()
q = "INSERT INTO clusters (user_id, name) VALUES (%s, %s)"
cursor.execute(q, (user_id, name))
cluster_id = cursor.lastrowid
self.cnx.commit()
cursor.close()
return cluster_id
def set_master_address(self, cluster_id, address):
cursor = self.cnx.cursor()
q = "UPDATE clusters SET master_address=%s WHERE clusters.id=%s"
print(address, cluster_id)
cursor.execute(q, (address, cluster_id))
self.cnx.commit()
cursor.close()
def new_container(self, cluster_id, user_id, docker_id, ip_address, contents):
cursor = self.cnx.cursor()
q = "INSERT INTO containers (user_id, cluster_id, docker_id, ip_address, contents) VALUES (%s, %s, %s)"
cursor.execute(q, (user_id, cluster_id, docker_id, ip_address, contents))
cont_id = cursor.lastrowid
self.cnx.commit()
cursor.close()
return cont_id
def new_proxy_entry(self, proxy_id, cluster_id, address, proxy_type, container_id):
cursor = self.cnx.cursor()
q = "INSERT INTO proxy (proxy_id, url, cluster_id, proxy_type, container_id) VALUES (%s, %s, %s)"
cursor.execute(q, (proxy_id, address, cluster_id, proxy_type, container_id))
self.cnx.commit()
cursor.close()
return proxy_id
def new_notebook(self, cluster_id, address, user_id, container_id):
cursor = self.cnx.cursor()
q = "INSERT INTO notebooks (cluster_id, address, user_id, container_id) VALUES (%s, %s, %s)"
cursor.execute(q, (cluster_id, address, user_id, container_id))
nb_id = cursor.lastrowid
self.cnx.commit()
cursor.close()
return nb_id
from docker import Client from docker import Client
from docker import errors as docker_errors
from docker.utils import create_host_config
from threading import Thread from threading import Thread
import time import time
from uuid import uuid4 as uuid
from caaas import get_db
REGISTRY = "10.0.0.2:5000"
MASTER_IMAGE = REGISTRY + "/venza/spark-master:1.4.1"
WORKER_IMAGE = REGISTRY + "/venza/spark-worker:1.4.1"
SHELL_IMAGE = REGISTRY + "/venza/spark-shell:1.4.1"
SUBMIT_IMAGE = REGISTRY + "/venza/spark-submit:1.4.1"
CONTAINER_IMAGE = REGISTRY + "/venza/spark-notebook:1.4.1"
def get_uuid():
return str(uuid())
class SwarmStatus: class SwarmStatus:
...@@ -36,4 +53,85 @@ class Swarm: ...@@ -36,4 +53,85 @@ class Swarm:
self.status.num_containers = info["Containers"] self.status.num_containers = info["Containers"]
self.status.num_nodes = info["DriverStatus"][3][1] self.status.num_nodes = info["DriverStatus"][3][1]
def get_notebook(self, user_id):
db = get_db()
nb = db.get_notebook(user_id)
if nb is None:
self.start_cluster_with_notebook(user_id)
nb = db.get_notebook(user_id)
return nb["address"]
def start_cluster_with_notebook(self, user_id):
num_workers = 2
self._create_new_spark_cluster(user_id, "notebook", num_workers, with_notebook=True)
def _create_new_spark_cluster(self, user_id, name, num_workers, with_notebook):
db = get_db()
try:
cluster_id = db.new_cluster(user_id, name)
master_info = self._spawn_spark_master(cluster_id, user_id)
db.set_master_address(cluster_id, master_info["spark_master_address"])
for i in range(num_workers):
self._spawn_spark_worker(cluster_id, user_id, master_info)
if with_notebook:
self._spawn_spark_notebook(cluster_id, user_id, master_info)
except docker_errors.APIError as e:
print("Error starting container: " + str(e.explanation))
# FIXME: should rollback all changes to DB
def _spawn_spark_master(self, cluster_id, user_id):
db = get_db()
options = {
"environment": {},
}
info = self._spawn_container(MASTER_IMAGE, options)
info["spark_master_address"] = "http://" + info["docker_ip"] + ":8080"
cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-master")
db.new_proxy_entry(get_uuid(), cluster_id, info["spark_master_address"], "spark-master", cont_id)
return info
def _spawn_spark_worker(self, cluster_id, user_id, master_info):
db = get_db()
options = {
"environment": {
"SPARK_MASTER_IP": master_info["docker_ip"],
"SPARK_WORKER_RAM": "4g",
"SPARK_WORKER_CORES": "2"
},
}
info = self._spawn_container(WORKER_IMAGE, options)
cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-worker")
db.new_proxy_entry(get_uuid(), cluster_id, "http://" + info["docker_ip"] + ":8081", "spark-worker", cont_id)
return info
def _spawn_spark_notebook(self, cluster_id, user_id, master_info):
db = get_db()
proxy_id = get_uuid()
options = {
"environment": {
"SPARK_MASTER_IP": master_info["docker_ip"],
"PROXY_ID": proxy_id
},
}
info = self._spawn_container(CONTAINER_IMAGE, options)
cont_id = db.new_container(cluster_id, user_id, info["docker_id"], info["docker_ip"], "spark-notebook")
db.new_proxy_entry(proxy_id, cluster_id, "http://" + info["docker_ip"] + ":9000/proxy/" + proxy_id, "spark-notebook", cont_id)
db.new_notebook(cluster_id, "http://bigfoot-m2.eurecom.fr/proxy/" + proxy_id, user_id, cont_id)
return info
def _spawn_container(self, image, options):
host_config = create_host_config(network_mode="bridge")
cont = self.cli.create_container(image=image,
environment=options["environment"],
network_disabled=False,
host_config=host_config,
detach=True)
self.cli.start(container=cont.get('Id'))
docker_info = self.cli.inspect_container(container=cont.get('Id'))
info = {
"docker_id": cont.get("Id"),
"docker_ip": docker_info["NetworkSettings"]["IPAddress"]
}
return info
swarm = Swarm() swarm = Swarm()
...@@ -10,7 +10,12 @@ ...@@ -10,7 +10,12 @@
<p>Monitor my activities:</p> <p>Monitor my activities:</p>
<ul> <ul>
<li>Running applications</li> <li><a href="/web/{{ user }}/status">Running applications</a></li>
</ul>
<p>Cluster status:</p>
<ul>
<li><a href="/web/status">Overview</a></li>
</ul> </ul>
{% endblock %} {% endblock %}
\ No newline at end of file
{% extends "base.html" %}
{% block title %}Spark notebook{% endblock %}
{% block content %}
<h1>Notebook</h1>
<p>You notebook is available at: <a href="{{ notebook_address }}">{{ notebook_address }}</a></p>
{% endblock %}
\ No newline at end of file
{% extends "base.html" %}
{% block title %}Status{% endblock %}
{% block content %}
<script type="application/javascript">
function gen_one_user(i, item) {
var block = $("<div></div>");
block.append("<p></p>").text(i);
var ul = $("<ul></ul>");
ul.append($("<li></li>").text("Num. clusters: " + item.num_clusters));
ul.append($("<li></li>").text("Num. containers: " + item.num_containers));
ul.append($("<li></li>").text("Has a notebook: " + item.has_notebook));
block.append(ul);
return block;
}
(function() {
$.getJSON( api_endpoint + "/full-status")
.done(function( data ) {
$.each(data, function(i, item) {
$("#userlist").append(gen_one_user(i, item))
})
});
})();
</script>
<h1>System status overview</h1>
<div id="userlist"></div>
{% endblock %}
\ No newline at end of file
from flask import render_template from flask import render_template
from caaas import app from caaas import app, swarm, get_db
from caaas.swarm import swarm
@app.route("/web/<username>") @app.route("/web/<username>")
...@@ -14,4 +13,14 @@ def web_index(username): ...@@ -14,4 +13,14 @@ def web_index(username):
@app.route("/web/<username>/spark-notebook") @app.route("/web/<username>/spark-notebook")
def web_notebook(username): def web_notebook(username):
pass user_id = get_db().get_user_id(username)
template_vars = {
"user": username,
"notebook_address": swarm.get_notebook(user_id)
}
return render_template('notebook.html', **template_vars)
@app.route("/web/status")
def web_status():
return render_template('status.html')
...@@ -3,13 +3,12 @@ ...@@ -3,13 +3,12 @@
SPARK_VER=1.4.1 SPARK_VER=1.4.1
HADOOP_VER=hadoop2.4 HADOOP_VER=hadoop2.4
./gen_dockerfiles.py $SPARK_VER $HADOOP_VER python ./gen_dockerfiles.py $SPARK_VER $HADOOP_VER
for d in master worker shell submit; do for d in master worker shell submit notebook; do