Commit 019ceab1 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Start preparing AML lab

parent 3c7dcda3
{
"name": "aml-lab",
"priority": 512,
"requires_binary": false,
"requires_binary": true,
"services": [
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-master",
"docker_image": "docker-registry:5000/zapps/spark2-master",
"environment": [
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
"{dns_name#self}"
],
[
"HADOOP_USER_NAME",
"{user_name}"
],
[
"PYTHONHASHSEED",
"42"
]
],
"essential_count": 1,
"monitor": false,
"name": "spark-master",
"networks": [],
"ports": [
{
"is_main_endpoint": false,
"expose": true,
"is_main_endpoint": true,
"name": "Spark master web interface",
"path": "/",
"port_number": 8080,
......@@ -25,10 +35,12 @@
],
"required_resources": {
"memory": 536870912
}
},
"startup_order": 0,
"total_count": 1
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-worker",
"docker_image": "docker-registry:5000/zapps/spark2-worker",
"environment": [
[
"SPARK_WORKER_CORES",
......@@ -40,15 +52,24 @@
],
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
"{dns_name#spark-master0}"
],
[
"SPARK_LOCAL_IP",
"spark-worker0-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
"{dns_name#self}"
],
[
"PYTHONHASHSEED",
"42"
],
[
"HADOOP_USER_NAME",
"{user_name}"
]
],
"essential_count": 1,
"monitor": false,
"name": "spark-worker0",
"name": "spark-worker",
"networks": [],
"ports": [
{
......@@ -61,14 +82,16 @@
],
"required_resources": {
"memory": 12884901888
}
},
"startup_order": 1,
"total_count": 2
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-jupyter-notebook",
"docker_image": "docker-registry:5000/zapps/spark2-jupyter-notebook",
"environment": [
[
"SPARK_MASTER",
"spark://spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe:7077"
"spark://{dns_name#spark-master0}:7077"
],
[
"SPARK_EXECUTOR_RAM",
......@@ -78,15 +101,24 @@
"SPARK_DRIVER_RAM",
"2147483648"
],
[
"HADOOP_USER_NAME",
"{user_name}"
],
[
"NB_USER",
"{user_name}"
],
[
"PYTHONHASHSEED",
"42"
],
[
"NAMENODE_HOST",
"hdfs-namenode.hdfs"
"hdfs-namenode.zoe"
]
],
"essential_count": 1,
"monitor": true,
"name": "spark-jupyter",
"networks": [],
......@@ -99,6 +131,7 @@
"protocol": "http"
},
{
"expose": true,
"is_main_endpoint": true,
"name": "Jupyter Notebook interface",
"path": "/",
......@@ -106,47 +139,13 @@
"protocol": "http"
}
],
"required_resources": {
"memory": 4294967296
}
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-worker",
"environment": [
[
"SPARK_WORKER_CORES",
"6"
],
[
"SPARK_WORKER_RAM",
"11273240064"
],
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
],
[
"SPARK_LOCAL_IP",
"spark-worker1-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
]
],
"monitor": false,
"name": "spark-worker1",
"networks": [],
"ports": [
{
"is_main_endpoint": false,
"name": "Spark worker web interface",
"path": "/",
"port_number": 8081,
"protocol": "http"
}
],
"required_resources": {
"memory": 12884901888
}
},
"startup_order": 0,
"total_count": 1
}
],
"version": 1,
"version": 2,
"will_end": false
}
......@@ -57,7 +57,7 @@ def zoe_web_main() -> int:
app_settings = {
'static_path': os.path.join(os.path.dirname(__file__), "web", "static"),
'template_path': os.path.join(os.path.dirname(__file__), "web", "templates"),
# 'debug': args.debug
'debug': args.debug
}
app = Application(zoe_api.web.web_init(api_endpoint) + zoe_api.rest_api.api_init(api_endpoint), **app_settings)
JinjaApp.init_app(app)
......
......@@ -26,12 +26,8 @@ from zoe_lib.version import ZOE_API_VERSION, ZOE_VERSION
def web_init(api_endpoint) -> List[tornado.web.URLSpec]:
"""Flask init for the web interface."""
# def before_request():
# """Use the Flask global to hold the api endpoint reference."""
# g.api_endpoint = api_endpoint
"""Tornado init for the web interface."""
# web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')
route_args = {
'api_endpoint': api_endpoint
}
......
......@@ -48,38 +48,42 @@ class HomeWeb(ZoeRequestHandler):
"""Home page with authentication."""
uid, role = get_auth(self)
if role == 'user' or role == 'admin':
executions = self.api_endpoint.execution_list(uid, role)
executions = self.api_endpoint.execution_list(uid, role)
template_vars = {
'executions': sorted(executions, key=lambda e: e.id),
'is_admin': role == 'admin',
}
self.render('home_user.html', **template_vars)
template_vars = {
'executions': sorted(executions, key=lambda e: e.id),
'is_admin': role == 'admin',
}
self.render('home_user.html', **template_vars)
else:
template_vars = {
'refresh': randint(2, 8),
'execution_status': 'Please wait...',
'execution_urls': [],
}
app_descr = json.load(open('contrib/zoeapps/eurecom_aml_lab.json', 'r'))
execution = self.api_endpoint.execution_list(uid, role, name='aml-lab')
if len(execution) == 0 or execution[0]['status'] == 'terminated' or execution[0]['status'] == 'finished':
self.api_endpoint.execution_start(uid, role, 'aml-lab', app_descr)
template_vars['execution_status'] = 'submitted'
class HomeGuestWeb(ZoeRequestHandler):
"""Handler for guest accesses."""
@catch_exceptions
def get(self):
"""Home page with demo/code access."""
template_vars = {
'kind': 'normal'
}
return self.render('home_guest.html', **template_vars)
app_descr = json.load(open('contrib/zoeapps/eurecom_aml_lab.json', 'r'))
execution = self.api_endpoint.execution_list(uid, role, name='aml-lab')
if len(execution) == 0 or execution[0]['status'] == 'terminated' or execution[0]['status'] == 'finished':
self.api_endpoint.execution_start(uid, role, 'aml-lab', app_descr)
template_vars['execution_status'] = 'submitted'
return self.render('home_guest.html', **template_vars)
else:
execution = execution[0]
if execution['status'] != 'running':
template_vars['execution_status'] = execution['status']
return self.render('home_guest.html', **template_vars)
else:
execution = execution[0]
if execution['status'] != 'running':
template_vars['execution_status'] = execution['status']
return self.render('home_guest.html', **template_vars)
else:
template_vars['refresh'] = -1
template_vars['execution_status'] = execution['status']
# for c_id in execution['services']:
# c = cont_api.get(c_id)
# ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
# for p in c['ports']:
# template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path'])))
return self.render('home_guest.html', **template_vars)
template_vars['refresh'] = -1
template_vars['execution_status'] = execution['status']
# for c_id in execution['services']:
# c = cont_api.get(c_id)
# ip = list(c['ip_address'].values())[0] # FIXME how to decide which network is the right one?
# for p in c['ports']:
# template_vars['execution_urls'].append(('{}'.format(p['name']), '{}://{}:{}{}'.format(p['protocol'], ip, p['port_number'], p['path'])))
return self.render('home_guest.html', **template_vars)
......@@ -15,6 +15,6 @@
"""Versions."""
ZOE_VERSION = '0.20.0-beta'
ZOE_VERSION = '0.10.3-aml'
ZOE_API_VERSION = '0.6'
ZOE_APPLICATION_FORMAT_VERSION = 2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment