Commit 8e5ac4d7 authored by Daniele Venzano's avatar Daniele Venzano

Start refactoring web interface for v2

parent 434ad2d2
......@@ -110,4 +110,13 @@ class SparkSubmitApplication(SparkApplication):
class PlainApplication:
pass
id = None
name = None
required_resources = None
user_id = None
executions = None
type = None
master_image = None
worker_image = None
notebook_image = None
submit_image = None
......@@ -84,4 +84,16 @@ class SparkSubmitExecution(Execution):
class PlainExecution:
pass
id = None
name = None
assigned_resources = None
application_id = None
time_started = None
time_scheduled = None
time_finished = None
status = None
termination_notice = None
cluster_id = None
type = None
commandline = None
spark_opts = None
......@@ -24,4 +24,6 @@ class User(Base):
class PlainUser:
pass
id = None
email = None
......@@ -140,7 +140,7 @@ class ZoeClient:
return None
return self.server.application_status(application.id)
def spark_application_list(self, user_id) -> [PlainApplication]:
def application_list(self, user_id) -> [PlainApplication]:
try:
self.state.query(User).filter_by(id=user_id).one()
except NoResultFound:
......
from flask import Flask
from zoe_web.api import api
from zoe_web.web import web
app = Flask(__name__)
app.register_blueprint(web, url_prefix='/web')
app.register_blueprint(api, url_prefix='/web/api')
import zoe_web.web
import zoe_web.api
app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
from flask import Blueprint, abort
api = Blueprint('api', __name__)
@api.route('/status/basic')
def basic_status():
abort(404)
from flask import jsonify, request, send_file, abort
from flask import jsonify, request, send_file, abort, Blueprint
from zipfile import is_zipfile
from zoe_web import app
from zoe_web.sql import CAaaState
from zoe_web.spark_app_execution import application_submitted, setup_volume, AppHistory
from zoe_web.swarm_manager import sm
STATS_CACHING_EXPIRATION = 1 # seconds
zoeweb_api = Blueprint('zoeweb_api', __name__)
@app.route("/api/<int:user_id>/cluster/<int:cluster_id>/terminate")
def api_terminate_cluster(user_id, cluster_id):
......
from zipfile import ZipFile
import os
import shutil
from zoe_web.sql import CAaaState
from zoe_web.config_parser import config
class AppHistory:
def __init__(self, user_id):
self.base_path = config.history_storage_path
self.per_user_max_count = int(config.history_per_user_count)
self.user_id = str(user_id)
def _app_path(self, app_id):
return os.path.join(self.base_path, self.user_id, str(app_id))
def _delete_app_history(self, app_id):
app_path = self._app_path(app_id)
shutil.rmtree(app_path)
def cleanup(self):
state = CAaaState()
num_apps = state.count_apps_finished(self.user_id)
if num_apps > self.per_user_max_count:
app_id = state.remove_oldest_application(self.user_id)
self._delete_app_history(app_id)
def add_application_zip(self, app_id, file_data):
app_path = self._app_path(app_id)
if not os.path.exists(app_path):
os.makedirs(app_path)
file_data.save(os.path.join(app_path, "app.zip"))
def save_log(self, app_id, logname, log):
app_path = self._app_path(app_id)
assert os.path.exists(app_path)
zip_path = os.path.join(app_path, "logs.zip")
z = ZipFile(zip_path, mode="a")
z.writestr(logname + ".txt", log)
z.close()
def get_log_archive_path(self, app_id):
app_path = self._app_path(app_id)
zip_path = os.path.join(app_path, "logs.zip")
if not os.path.exists(app_path):
return None
else:
return zip_path
def application_submitted(user_id, execution_name, spark_options, commandline, file_data) -> int:
ah = AppHistory(user_id)
ah.cleanup()
state = CAaaState()
app_id = state.new_application(user_id, execution_name, spark_options, commandline)
ah.add_application_zip(app_id, file_data)
return app_id
def setup_volume(user_id, app_id, app_pkg):
app_pkg = ZipFile(app_pkg)
exec_path = config.docker_volume_path
exec_path = os.path.join(exec_path, str(user_id), str(app_id))
os.makedirs(exec_path)
app_pkg.extractall(exec_path)
state = CAaaState()
state.application_ready(app_id)
This diff is collapsed.
This diff is collapsed.
from uuid import uuid1 as uuid
def get_uuid():
return str(uuid())
from flask import Blueprint, abort, render_template
from zoe_client import ZoeClient
import zoe_web.web.utils as web_utils
web = Blueprint('web', __name__, template_folder='templates', static_folder='static')
@web.route('/')
def index():
return render_template('index.html')
@web.route('/home')
def home():
client = ZoeClient()
user = web_utils.check_user(client)
template_vars = {
"user_id": user.id,
"email": user.email
}
return render_template('home.html', **template_vars)
......@@ -3,12 +3,9 @@
<head>
{% block head %}
<meta charset="UTF-8">
<title>{% block title %}{% endblock %} - CAAAS</title>
<title>{% block title %}{% endblock %} - Zoe</title>
<script src="/static/jquery-2.1.4.min.js" type="application/javascript"></script>
<script type="application/javascript">
var api_endpoint = "/api";
</script>
<link rel="stylesheet" href="/static/caaas.css" type="text/css">
<link rel="stylesheet" href="/static/zoe.css" type="text/css">
{% endblock %}
</head>
<body>
......@@ -26,13 +23,13 @@
update_status();
window.setInterval(update_status, 5000);
</script>
<h1>DSG CAaaS system - Analytics on demand</h1>
<h1>Zoe - Analytics on demand</h1>
<div id="content">{% block content %}{% endblock %}</div>
<div id="footer">
{% block footer %}
<div class="caaas_status_line">
<div class="status_line">
<p>
<span id="app_name"><a href="{{ url_for("web_status") }}">CAaaS</a></span>
<span id="app_name"><a href="{{ url_for("web_status") }}">Zoe</a></span>
<span id="status">
[<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers]
</span>
......
from flask import session, redirect, url_for
from zoe_client import ZoeClient
from zoe_web.web import web
def check_user(zoeclient: ZoeClient):
if 'user_id' not in session:
return redirect(url_for(web.index))
if not zoeclient.user_check(session['user_id']):
return redirect(url_for('index'))
......@@ -2,11 +2,7 @@ from flask import render_template, redirect, url_for, abort
from zoe_web import app
from zoe_client import ZoeClient
@app.route("/web/")
def index():
return render_template('index.html')
from common.configuration import conf
@app.route("/web/status")
......@@ -32,7 +28,7 @@ def web_index(user_id):
return redirect(url_for('index'))
template_vars = {
"user_id": user_id,
"email": client.user_get_email(user_id)
"email": client.user_get(user_id).email
}
return render_template('home.html', **template_vars)
......@@ -43,7 +39,7 @@ def web_user_apps(user_id):
if not client.user_check(user_id):
return redirect(url_for('index'))
apps = client.spark_application_list(user_id)
apps = client.application_list(user_id)
template_vars = {
"user_id": user_id,
"apps": apps
......@@ -53,53 +49,48 @@ def web_user_apps(user_id):
@app.route("/web/<int:user_id>/spark-notebook")
def web_notebook(user_id):
state = CAaaState()
if not state.check_user_id(user_id):
client = ZoeClient()
if not client.user_check(user_id):
return redirect(url_for('index'))
template_vars = {
"user_id": user_id,
"notebook_address": sm.get_notebook(user_id),
"max_age": config.cleanup_notebooks_older_than,
"wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning)
"max_age": conf['notebook_max_age_no_activity'],
"wrn_time": conf['notebook_max_age_no_activity'] - conf['notebook_warning_age_no_activity']
}
return render_template('notebook.html', **template_vars)
@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/inspect")
def web_inspect(user_id, cluster_id):
state = CAaaState()
if not state.check_user_id(user_id):
@app.route("/web/<int:user_id>/cluster/<int:app_id>/inspect")
def web_inspect(user_id, app_id):
client = ZoeClient()
if not client.user_check(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id:
application = client.application_get(app_id)
if application.user_id != user_id:
abort(404)
containers = state.get_containers(cluster_id=cluster_id)
clist = []
for cid, cinfo in containers.items():
plist = get_container_addresses(cid)
clist.append([cinfo["contents"], plist, cid])
app_report = client.application_status(app_id)
template_vars = {
"cluster_name": cluster["name"],
"containers": clist,
"application": app_report,
"user_id": user_id
}
return render_template('inspect.html', **template_vars)
@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/terminate")
def web_terminate(user_id, cluster_id):
state = CAaaState()
if not state.check_user_id(user_id):
@app.route("/web/<int:user_id>/execution/<int:execution_id>/terminate")
def web_terminate(user_id, execution_id):
client = ZoeClient()
if not client.user_check(user_id):
return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id)
if cluster["user_id"] != user_id:
execution = client.execution_get(execution_id)
application = client.application_get(execution.application_id)
if application.user_id != user_id:
abort(404)
template_vars = {
"cluster_name": cluster["name"],
"cluster_id": cluster_id,
"execution_name": execution.name,
"execution_id": execution_id,
"user_id": user_id
}
return render_template('terminate.html', **template_vars)
......@@ -107,19 +98,18 @@ def web_terminate(user_id, cluster_id):
@app.route("/web/<int:user_id>/container/<int:container_id>/logs")
def web_logs(user_id, container_id):
state = CAaaState()
if not state.check_user_id(user_id):
client = ZoeClient()
if not client.user_check(user_id):
return redirect(url_for('index'))
cont = state.get_container(container_id)
cont = client.container_get(container_id)
if user_id != cont["user_id"]:
abort(404)
logs = sm.get_log(container_id)
logs = client.log_get(container_id)
if logs is None:
abort(404)
else:
logs = logs.decode("ascii")
ret = {
'user_id': user_id,
'cont_contents': cont['contents'],
......@@ -130,8 +120,8 @@ def web_logs(user_id, container_id):
@app.route("/web/<int:user_id>/submit-spark-app")
def web_spark_submit(user_id):
state = CAaaState()
if not state.check_user_id(user_id):
client = ZoeClient()
if not client.user_check(user_id):
return redirect(url_for('index'))
template_vars = {
......
......@@ -75,7 +75,7 @@ def app_rm_cmd(args):
e = client.execution_get(eid)
if e.status == "running":
print("Terminating execution {}".format(e.name))
client.execution_terminate(e)
client.execution_terminate(e.id)
client.application_remove(application.id)
......@@ -92,7 +92,7 @@ def app_inspect_cmd(args):
def app_list_cmd(args):
client = ZoeClient()
applications = client.spark_application_list(args.id)
applications = client.application_list(args.id)
if len(applications) > 0:
print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
for app in applications:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment