Commit 434956e4 authored by Daniele Venzano's avatar Daniele Venzano

The web interface now starts correctly

Implement missing status information parsing from Docker
parent b4c70be0
......@@ -6,9 +6,7 @@ from common.state import Application, Execution, SparkSubmitExecution
class Report:
def __init__(self):
self.report = {
'executions': []
}
self.report = {}
def __str__(self):
return pformat(self.report)
......@@ -16,14 +14,13 @@ class Report:
class PlatformStatusReport(Report):
def include_swarm_status(self, sw_status: SwarmStatus):
self.report["swarm"] = {
'total cores': sw_status.cores_total
}
self.report["swarm"] = sw_status.to_dict()
class ApplicationStatusReport(Report):
def __init__(self, application: Application):
super().__init__()
self.report["executions"] = []
self._app_to_dict(application)
def _app_to_dict(self, application: Application):
......
......@@ -17,6 +17,7 @@ from common.object_storage import logs_archive_upload
class PlatformManager:
def __init__(self):
self.swarm = SwarmClient()
pm.update_proxy()
def start_execution(self, execution_id: int, resources: ApplicationResources) -> bool:
state = AlchemySession()
......
......@@ -43,7 +43,9 @@ class RPyCAsyncIOServer:
self.auto_register = bool(registrar)
else:
self.auto_register = auto_register
self.protocol_config = protocol_config
self.protocol_config = {"allow_public_attrs": True}
if protocol_config is not None:
self.protocol_config.update(protocol_config)
self.hostname = hostname
self.port = port
......
......@@ -8,7 +8,7 @@ import docker.errors
from common.configuration import conf
from zoe_scheduler.swarm_status import SwarmStatus
from zoe_scheduler.swarm_status import SwarmStatus, SwarmNodeStatus
class SwarmClient:
......@@ -30,8 +30,23 @@ class SwarmClient:
pl_status.placement_strategy = info["DriverStatus"][idx][1]
idx = 2
assert 'Filters' in info["DriverStatus"][idx][0]
pl_status.active_filters = info["DriverStatus"][idx][1].split(", ")
pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")]
idx = 3
assert 'Nodes' in info["DriverStatus"][idx][0]
node_count = int(info["DriverStatus"][idx][1])
idx = 4
for node in range(node_count):
ns = SwarmNodeStatus(info["DriverStatus"][idx + node][0])
ns.docker_endpoint = info["DriverStatus"][idx + node][1]
ns.container_count = int(info["DriverStatus"][idx + node + 1][1])
ns.cores_reserved = int(info["DriverStatus"][idx + node + 2][1].split(' / ')[0])
ns.cores_total = int(info["DriverStatus"][idx + node + 2][1].split(' / ')[1])
ns.memory_reserved = info["DriverStatus"][idx + node + 3][1].split(' / ')[0]
ns.memory_total = info["DriverStatus"][idx + node + 3][1].split(' / ')[1]
ns.labels = ns.cores_reserved = info["DriverStatus"][idx + node + 4][1:]
pl_status.nodes.append(ns)
idx += 4
pl_status.timestamp = time.time()
return pl_status
......
......@@ -12,6 +12,18 @@ class SwarmNodeStatus:
self.memory_reserved = 0
self.labels = {}
def to_dict(self):
return {
'name': self.name,
'docker_endpoint': self.docker_endpoint,
'container_count': self.container_count,
'cores_total': self.cores_total,
'cores_reserved': self.cores_reserved,
'memory_total': self.memory_total,
'memory_reserved': self.memory_reserved,
'labels': self.labels.copy()
}
class SwarmStatus:
def __init__(self):
......@@ -23,3 +35,16 @@ class SwarmStatus:
self.active_filters = []
self.nodes = []
self.timestamp = time.time()
def to_dict(self):
ret = {
'container_count': self.container_count,
'image_count': self.image_count,
'memory_total': self.memory_total,
'cores_total': self.cores_total,
'placement_strategy': self.placement_strategy,
'active_filters': self.active_filters.copy(),
'timestamp': self.timestamp,
'nodes': [x.to_dict() for x in self.nodes]
}
return ret
from flask import Flask
from flask import Flask, url_for, abort
from zoe_web.api import api_bp
from zoe_web.web import web_bp
app = Flask(__name__)
app.register_blueprint(web_bp, url_prefix='/web')
app.register_blueprint(api_bp, url_prefix='/web/api')
app = Flask(__name__, static_url_path='/does-not-exist')
app.register_blueprint(web_bp, url_prefix='')
app.register_blueprint(api_bp, url_prefix='/api')
app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
def debug_list_routes():
output = []
for rule in app.url_map.iter_rules():
options = {}
for arg in rule.arguments:
options[arg] = "[{0}]".format(arg)
methods = ','.join(rule.methods)
url = url_for(rule.endpoint, **options)
line = "{:50s} {:20s} {}".format(rule.endpoint, methods, url)
output.append(line)
for line in sorted(output):
print(line)
@app.errorhandler(404)
def page_not_found(_):
debug_list_routes()
from flask import Blueprint, abort
from flask import Blueprint, abort, jsonify
api_bp = Blueprint('api', __name__)
from zoe_client import get_zoe_client
@api_bp.route('/status/basic')
def basic_status():
abort(404)
def status_basic():
client = get_zoe_client()
platform_report = client.platform_status()
ret = {
'num_nodes': len(platform_report.report["swarm"]["nodes"]),
'num_containers': platform_report.report["swarm"]["container_count"]
}
return jsonify(**ret)
from flask import session, redirect, url_for
from zoe_client import ZoeClient
from zoe_web.web import web_bp
from common.configuration import conf
def get_zoe_client():
if conf['client_rpyc_autodiscovery']:
return ZoeClient()
else:
return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])
def check_user(zoeclient: ZoeClient):
if 'user_id' not in session:
return redirect(url_for(web_bp.index))
return redirect(url_for('web_bp.index'))
if not zoeclient.user_check(session['user_id']):
return redirect(url_for('index'))
return redirect(url_for('web_bp.index'))
from flask import Blueprint
web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')
import zoe_web.web.start
import zoe_web.web.status
......@@ -2,7 +2,7 @@ from flask import render_template
from zoe_client import get_zoe_client
from zoe_web.web import web_bp
import zoe_web.web.utils as web_utils
import zoe_web.utils as web_utils
@web_bp.route('/')
......
from flask import render_template
from zoe_client import get_zoe_client
from zoe_web.web import web_bp
import zoe_web.utils as web_utils
@web_bp.route('/status/platform')
def status_platform():
client = get_zoe_client()
user = web_utils.check_user(client)
template_vars = {
"user_id": user.id,
"email": user.email
}
return render_template('status.html', **template_vars)
......@@ -11,7 +11,7 @@
<body>
<script>
function update_status() {
$.getJSON("{{ url_for("api_status") }}")
$.getJSON("{{ url_for("api.status_basic") }}")
.done(function( data ) {
$("#num_nodes").text(data.num_nodes);
$("#num_containers").text(data.num_containers);
......@@ -29,7 +29,7 @@
{% block footer %}
<div class="status_line">
<p>
<span id="app_name"><a href="{{ url_for("web_status") }}">Zoe</a></span>
<span id="app_name"><a href="{{ url_for("web.status_platform") }}">Zoe</a></span>
<span id="status">
[<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers]
</span>
......
......@@ -30,7 +30,7 @@ def user_new_cmd(args):
def user_get_cmd(args):
client = get_zoe_client()
user = client.user_get(args.email)
print("User ID: {}".format(user.email))
print("User ID: {}".format(user.id))
def spark_cluster_new_cmd(args):
......@@ -216,7 +216,4 @@ def main():
args.func(args)
if __name__ == "__main__":
try:
main()
except AttributeError:
argparser.print_help()
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment