[GITLAB] - UPGRADE TO v12 on Wednesday the 18th of December at 11.30AM

Commit 8e5ac4d7 authored by Daniele Venzano's avatar Daniele Venzano

Start refactoring web interface for v2

parent 434ad2d2
...@@ -110,4 +110,13 @@ class SparkSubmitApplication(SparkApplication): ...@@ -110,4 +110,13 @@ class SparkSubmitApplication(SparkApplication):
class PlainApplication: class PlainApplication:
pass id = None
name = None
required_resources = None
user_id = None
executions = None
type = None
master_image = None
worker_image = None
notebook_image = None
submit_image = None
...@@ -84,4 +84,16 @@ class SparkSubmitExecution(Execution): ...@@ -84,4 +84,16 @@ class SparkSubmitExecution(Execution):
class PlainExecution: class PlainExecution:
pass id = None
name = None
assigned_resources = None
application_id = None
time_started = None
time_scheduled = None
time_finished = None
status = None
termination_notice = None
cluster_id = None
type = None
commandline = None
spark_opts = None
...@@ -24,4 +24,6 @@ class User(Base): ...@@ -24,4 +24,6 @@ class User(Base):
class PlainUser: class PlainUser:
pass id = None
email = None
...@@ -140,7 +140,7 @@ class ZoeClient: ...@@ -140,7 +140,7 @@ class ZoeClient:
return None return None
return self.server.application_status(application.id) return self.server.application_status(application.id)
def spark_application_list(self, user_id) -> [PlainApplication]: def application_list(self, user_id) -> [PlainApplication]:
try: try:
self.state.query(User).filter_by(id=user_id).one() self.state.query(User).filter_by(id=user_id).one()
except NoResultFound: except NoResultFound:
......
from flask import Flask from flask import Flask
from zoe_web.api import api
from zoe_web.web import web
app = Flask(__name__) app = Flask(__name__)
app.register_blueprint(web, url_prefix='/web')
app.register_blueprint(api, url_prefix='/web/api')
import zoe_web.web app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
import zoe_web.api
from flask import Blueprint, abort
api = Blueprint('api', __name__)
@api.route('/status/basic')
def basic_status():
abort(404)
from flask import jsonify, request, send_file, abort from flask import jsonify, request, send_file, abort, Blueprint
from zipfile import is_zipfile from zipfile import is_zipfile
from zoe_web import app
from zoe_web.sql import CAaaState
from zoe_web.spark_app_execution import application_submitted, setup_volume, AppHistory
from zoe_web.swarm_manager import sm
STATS_CACHING_EXPIRATION = 1 # seconds
zoeweb_api = Blueprint('zoeweb_api', __name__)
@app.route("/api/<int:user_id>/cluster/<int:cluster_id>/terminate") @app.route("/api/<int:user_id>/cluster/<int:cluster_id>/terminate")
def api_terminate_cluster(user_id, cluster_id): def api_terminate_cluster(user_id, cluster_id):
......
from zipfile import ZipFile
import os
import shutil
from zoe_web.sql import CAaaState
from zoe_web.config_parser import config
class AppHistory:
def __init__(self, user_id):
self.base_path = config.history_storage_path
self.per_user_max_count = int(config.history_per_user_count)
self.user_id = str(user_id)
def _app_path(self, app_id):
return os.path.join(self.base_path, self.user_id, str(app_id))
def _delete_app_history(self, app_id):
app_path = self._app_path(app_id)
shutil.rmtree(app_path)
def cleanup(self):
state = CAaaState()
num_apps = state.count_apps_finished(self.user_id)
if num_apps > self.per_user_max_count:
app_id = state.remove_oldest_application(self.user_id)
self._delete_app_history(app_id)
def add_application_zip(self, app_id, file_data):
app_path = self._app_path(app_id)
if not os.path.exists(app_path):
os.makedirs(app_path)
file_data.save(os.path.join(app_path, "app.zip"))
def save_log(self, app_id, logname, log):
app_path = self._app_path(app_id)
assert os.path.exists(app_path)
zip_path = os.path.join(app_path, "logs.zip")
z = ZipFile(zip_path, mode="a")
z.writestr(logname + ".txt", log)
z.close()
def get_log_archive_path(self, app_id):
app_path = self._app_path(app_id)
zip_path = os.path.join(app_path, "logs.zip")
if not os.path.exists(app_path):
return None
else:
return zip_path
def application_submitted(user_id, execution_name, spark_options, commandline, file_data) -> int:
ah = AppHistory(user_id)
ah.cleanup()
state = CAaaState()
app_id = state.new_application(user_id, execution_name, spark_options, commandline)
ah.add_application_zip(app_id, file_data)
return app_id
def setup_volume(user_id, app_id, app_pkg):
app_pkg = ZipFile(app_pkg)
exec_path = config.docker_volume_path
exec_path = os.path.join(exec_path, str(user_id), str(app_id))
os.makedirs(exec_path)
app_pkg.extractall(exec_path)
state = CAaaState()
state.application_ready(app_id)
This diff is collapsed.
This diff is collapsed.
from uuid import uuid1 as uuid
def get_uuid():
return str(uuid())
from flask import Blueprint, abort, render_template
from zoe_client import ZoeClient
import zoe_web.web.utils as web_utils
web = Blueprint('web', __name__, template_folder='templates', static_folder='static')
@web.route('/')
def index():
return render_template('index.html')
@web.route('/home')
def home():
client = ZoeClient()
user = web_utils.check_user(client)
template_vars = {
"user_id": user.id,
"email": user.email
}
return render_template('home.html', **template_vars)
...@@ -3,12 +3,9 @@ ...@@ -3,12 +3,9 @@
<head> <head>
{% block head %} {% block head %}
<meta charset="UTF-8"> <meta charset="UTF-8">
<title>{% block title %}{% endblock %} - CAAAS</title> <title>{% block title %}{% endblock %} - Zoe</title>
<script src="/static/jquery-2.1.4.min.js" type="application/javascript"></script> <script src="/static/jquery-2.1.4.min.js" type="application/javascript"></script>
<script type="application/javascript"> <link rel="stylesheet" href="/static/zoe.css" type="text/css">
var api_endpoint = "/api";
</script>
<link rel="stylesheet" href="/static/caaas.css" type="text/css">
{% endblock %} {% endblock %}
</head> </head>
<body> <body>
...@@ -26,13 +23,13 @@ ...@@ -26,13 +23,13 @@
update_status(); update_status();
window.setInterval(update_status, 5000); window.setInterval(update_status, 5000);
</script> </script>
<h1>DSG CAaaS system - Analytics on demand</h1> <h1>Zoe - Analytics on demand</h1>
<div id="content">{% block content %}{% endblock %}</div> <div id="content">{% block content %}{% endblock %}</div>
<div id="footer"> <div id="footer">
{% block footer %} {% block footer %}
<div class="caaas_status_line"> <div class="status_line">
<p> <p>
<span id="app_name"><a href="{{ url_for("web_status") }}">CAaaS</a></span> <span id="app_name"><a href="{{ url_for("web_status") }}">Zoe</a></span>
<span id="status"> <span id="status">
[<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers] [<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers]
</span> </span>
......
from flask import session, redirect, url_for
from zoe_client import ZoeClient
from zoe_web.web import web
def check_user(zoeclient: ZoeClient):
if 'user_id' not in session:
return redirect(url_for(web.index))
if not zoeclient.user_check(session['user_id']):
return redirect(url_for('index'))
...@@ -2,11 +2,7 @@ from flask import render_template, redirect, url_for, abort ...@@ -2,11 +2,7 @@ from flask import render_template, redirect, url_for, abort
from zoe_web import app from zoe_web import app
from zoe_client import ZoeClient from zoe_client import ZoeClient
from common.configuration import conf
@app.route("/web/")
def index():
return render_template('index.html')
@app.route("/web/status") @app.route("/web/status")
...@@ -32,7 +28,7 @@ def web_index(user_id): ...@@ -32,7 +28,7 @@ def web_index(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
template_vars = { template_vars = {
"user_id": user_id, "user_id": user_id,
"email": client.user_get_email(user_id) "email": client.user_get(user_id).email
} }
return render_template('home.html', **template_vars) return render_template('home.html', **template_vars)
...@@ -43,7 +39,7 @@ def web_user_apps(user_id): ...@@ -43,7 +39,7 @@ def web_user_apps(user_id):
if not client.user_check(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
apps = client.spark_application_list(user_id) apps = client.application_list(user_id)
template_vars = { template_vars = {
"user_id": user_id, "user_id": user_id,
"apps": apps "apps": apps
...@@ -53,53 +49,48 @@ def web_user_apps(user_id): ...@@ -53,53 +49,48 @@ def web_user_apps(user_id):
@app.route("/web/<int:user_id>/spark-notebook") @app.route("/web/<int:user_id>/spark-notebook")
def web_notebook(user_id): def web_notebook(user_id):
state = CAaaState() client = ZoeClient()
if not state.check_user_id(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
template_vars = { template_vars = {
"user_id": user_id, "user_id": user_id,
"notebook_address": sm.get_notebook(user_id), "max_age": conf['notebook_max_age_no_activity'],
"max_age": config.cleanup_notebooks_older_than, "wrn_time": conf['notebook_max_age_no_activity'] - conf['notebook_warning_age_no_activity']
"wrn_time": int(config.cleanup_notebooks_older_than) - int(config.cleanup_notebooks_warning)
} }
return render_template('notebook.html', **template_vars) return render_template('notebook.html', **template_vars)
@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/inspect") @app.route("/web/<int:user_id>/cluster/<int:app_id>/inspect")
def web_inspect(user_id, cluster_id): def web_inspect(user_id, app_id):
state = CAaaState() client = ZoeClient()
if not state.check_user_id(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id) application = client.application_get(app_id)
if cluster["user_id"] != user_id: if application.user_id != user_id:
abort(404) abort(404)
containers = state.get_containers(cluster_id=cluster_id) app_report = client.application_status(app_id)
clist = []
for cid, cinfo in containers.items():
plist = get_container_addresses(cid)
clist.append([cinfo["contents"], plist, cid])
template_vars = { template_vars = {
"cluster_name": cluster["name"], "application": app_report,
"containers": clist,
"user_id": user_id "user_id": user_id
} }
return render_template('inspect.html', **template_vars) return render_template('inspect.html', **template_vars)
@app.route("/web/<int:user_id>/cluster/<int:cluster_id>/terminate") @app.route("/web/<int:user_id>/execution/<int:execution_id>/terminate")
def web_terminate(user_id, cluster_id): def web_terminate(user_id, execution_id):
state = CAaaState() client = ZoeClient()
if not state.check_user_id(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
cluster = state.get_cluster(cluster_id) execution = client.execution_get(execution_id)
if cluster["user_id"] != user_id: application = client.application_get(execution.application_id)
if application.user_id != user_id:
abort(404) abort(404)
template_vars = { template_vars = {
"cluster_name": cluster["name"], "execution_name": execution.name,
"cluster_id": cluster_id, "execution_id": execution_id,
"user_id": user_id "user_id": user_id
} }
return render_template('terminate.html', **template_vars) return render_template('terminate.html', **template_vars)
...@@ -107,19 +98,18 @@ def web_terminate(user_id, cluster_id): ...@@ -107,19 +98,18 @@ def web_terminate(user_id, cluster_id):
@app.route("/web/<int:user_id>/container/<int:container_id>/logs") @app.route("/web/<int:user_id>/container/<int:container_id>/logs")
def web_logs(user_id, container_id): def web_logs(user_id, container_id):
state = CAaaState() client = ZoeClient()
if not state.check_user_id(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
cont = state.get_container(container_id) cont = client.container_get(container_id)
if user_id != cont["user_id"]: if user_id != cont["user_id"]:
abort(404) abort(404)
logs = sm.get_log(container_id) logs = client.log_get(container_id)
if logs is None: if logs is None:
abort(404) abort(404)
else: else:
logs = logs.decode("ascii")
ret = { ret = {
'user_id': user_id, 'user_id': user_id,
'cont_contents': cont['contents'], 'cont_contents': cont['contents'],
...@@ -130,8 +120,8 @@ def web_logs(user_id, container_id): ...@@ -130,8 +120,8 @@ def web_logs(user_id, container_id):
@app.route("/web/<int:user_id>/submit-spark-app") @app.route("/web/<int:user_id>/submit-spark-app")
def web_spark_submit(user_id): def web_spark_submit(user_id):
state = CAaaState() client = ZoeClient()
if not state.check_user_id(user_id): if not client.user_check(user_id):
return redirect(url_for('index')) return redirect(url_for('index'))
template_vars = { template_vars = {
......
...@@ -75,7 +75,7 @@ def app_rm_cmd(args): ...@@ -75,7 +75,7 @@ def app_rm_cmd(args):
e = client.execution_get(eid) e = client.execution_get(eid)
if e.status == "running": if e.status == "running":
print("Terminating execution {}".format(e.name)) print("Terminating execution {}".format(e.name))
client.execution_terminate(e) client.execution_terminate(e.id)
client.application_remove(application.id) client.application_remove(application.id)
...@@ -92,7 +92,7 @@ def app_inspect_cmd(args): ...@@ -92,7 +92,7 @@ def app_inspect_cmd(args):
def app_list_cmd(args): def app_list_cmd(args):
client = ZoeClient() client = ZoeClient()
applications = client.spark_application_list(args.id) applications = client.application_list(args.id)
if len(applications) > 0: if len(applications) > 0:
print("{:4} {:20} {:25}".format("ID", "Name", "Type")) print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
for app in applications: for app in applications:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment