The web interface now starts correctly

Implement missing status information parsing from Docker

common/status.py

@@ -6,9 +6,7 @@ from common.state import Application, Execution, SparkSubmitExecution
 
 class Report:
     def __init__(self):
-        self.report = {
-            'executions': []
-        }
+        self.report = {}
 
     def __str__(self):
         return pformat(self.report)
@@ -16,14 +14,13 @@ class Report:
 
 class PlatformStatusReport(Report):
     def include_swarm_status(self, sw_status: SwarmStatus):
-        self.report["swarm"] = {
-            'total cores': sw_status.cores_total
-        }
+        self.report["swarm"] = sw_status.to_dict()
 
 
 class ApplicationStatusReport(Report):
     def __init__(self, application: Application):
         super().__init__()
+        self.report["executions"] = []
         self._app_to_dict(application)
 
     def _app_to_dict(self, application: Application):

zoe_scheduler/platform.py

@@ -17,6 +17,7 @@ from common.object_storage import logs_archive_upload
 class PlatformManager:
     def __init__(self):
         self.swarm = SwarmClient()
+        pm.update_proxy()
 
     def start_execution(self, execution_id: int, resources: ApplicationResources) -> bool:
         state = AlchemySession()

zoe_scheduler/rpyc_server.py

@@ -43,7 +43,9 @@ class RPyCAsyncIOServer:
             self.auto_register = bool(registrar)
         else:
             self.auto_register = auto_register
-        self.protocol_config = protocol_config
+        self.protocol_config = {"allow_public_attrs": True}
+        if protocol_config is not None:
+            self.protocol_config.update(protocol_config)
 
         self.hostname = hostname
         self.port = port

zoe_scheduler/swarm_client.py

@@ -8,7 +8,7 @@ import docker.errors
 
 from common.configuration import conf
 
-from zoe_scheduler.swarm_status import SwarmStatus
+from zoe_scheduler.swarm_status import SwarmStatus, SwarmNodeStatus
 
 
 class SwarmClient:
@@ -30,8 +30,23 @@ class SwarmClient:
         pl_status.placement_strategy = info["DriverStatus"][idx][1]
         idx = 2
         assert 'Filters' in info["DriverStatus"][idx][0]
-        pl_status.active_filters = info["DriverStatus"][idx][1].split(", ")
-
+        pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")]
+        idx = 3
+        assert 'Nodes' in info["DriverStatus"][idx][0]
+        node_count = int(info["DriverStatus"][idx][1])
+        idx = 4
+        for node in range(node_count):
+            ns = SwarmNodeStatus(info["DriverStatus"][idx + node][0])
+            ns.docker_endpoint = info["DriverStatus"][idx + node][1]
+            ns.container_count = int(info["DriverStatus"][idx + node + 1][1])
+            ns.cores_reserved = int(info["DriverStatus"][idx + node + 2][1].split(' / ')[0])
+            ns.cores_total = int(info["DriverStatus"][idx + node + 2][1].split(' / ')[1])
+            ns.memory_reserved = info["DriverStatus"][idx + node + 3][1].split(' / ')[0]
+            ns.memory_total = info["DriverStatus"][idx + node + 3][1].split(' / ')[1]
+            ns.labels = ns.cores_reserved = info["DriverStatus"][idx + node + 4][1:]
+
+            pl_status.nodes.append(ns)
+            idx += 4
         pl_status.timestamp = time.time()
         return pl_status
 

zoe_scheduler/swarm_status.py

@@ -12,6 +12,18 @@ class SwarmNodeStatus:
         self.memory_reserved = 0
         self.labels = {}
 
+    def to_dict(self):
+        return {
+            'name': self.name,
+            'docker_endpoint': self.docker_endpoint,
+            'container_count': self.container_count,
+            'cores_total': self.cores_total,
+            'cores_reserved': self.cores_reserved,
+            'memory_total': self.memory_total,
+            'memory_reserved': self.memory_reserved,
+            'labels': self.labels.copy()
+        }
+
 
 class SwarmStatus:
     def __init__(self):
@@ -23,3 +35,16 @@ class SwarmStatus:
         self.active_filters = []
         self.nodes = []
         self.timestamp = time.time()
+
+    def to_dict(self):
+        ret = {
+            'container_count': self.container_count,
+            'image_count': self.image_count,
+            'memory_total': self.memory_total,
+            'cores_total': self.cores_total,
+            'placement_strategy': self.placement_strategy,
+            'active_filters': self.active_filters.copy(),
+            'timestamp': self.timestamp,
+            'nodes': [x.to_dict() for x in self.nodes]
+        }
+        return ret

zoe_web/__init__.py

-from flask import Flask
+from flask import Flask, url_for, abort
 from zoe_web.api import api_bp
 from zoe_web.web import web_bp
 
-app = Flask(__name__)
-app.register_blueprint(web_bp, url_prefix='/web')
-app.register_blueprint(api_bp, url_prefix='/web/api')
+app = Flask(__name__, static_url_path='/does-not-exist')
+app.register_blueprint(web_bp, url_prefix='')
+app.register_blueprint(api_bp, url_prefix='/api')
 
 app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
+
+
+def debug_list_routes():
+    output = []
+    for rule in app.url_map.iter_rules():
+
+        options = {}
+        for arg in rule.arguments:
+            options[arg] = "[{0}]".format(arg)
+
+        methods = ','.join(rule.methods)
+        url = url_for(rule.endpoint, **options)
+        line = "{:50s} {:20s} {}".format(rule.endpoint, methods, url)
+        output.append(line)
+
+    for line in sorted(output):
+        print(line)
+
+
+@app.errorhandler(404)
+def page_not_found(_):
+    debug_list_routes()

zoe_web/api/__init__.py

-from flask import Blueprint, abort
-
+from flask import Blueprint, abort, jsonify
 api_bp = Blueprint('api', __name__)
 
+from zoe_client import get_zoe_client
+
 
 @api_bp.route('/status/basic')
-def basic_status():
-    abort(404)
+def status_basic():
+    client = get_zoe_client()
+    platform_report = client.platform_status()
+    ret = {
+        'num_nodes': len(platform_report.report["swarm"]["nodes"]),
+        'num_containers': platform_report.report["swarm"]["container_count"]
+    }
+    return jsonify(**ret)

zoe_web/web/utils.py → zoe_web/utils.py

 from flask import session, redirect, url_for
 
 from zoe_client import ZoeClient
-from zoe_web.web import web_bp
-from common.configuration import conf
-
-
-def get_zoe_client():
-    if conf['client_rpyc_autodiscovery']:
-        return ZoeClient()
-    else:
-        return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])
 
 
 def check_user(zoeclient: ZoeClient):
     if 'user_id' not in session:
-        return redirect(url_for(web_bp.index))
+        return redirect(url_for('web_bp.index'))
     if not zoeclient.user_check(session['user_id']):
-        return redirect(url_for('index'))
+        return redirect(url_for('web_bp.index'))

zoe_web/web/__init__.py

 from flask import Blueprint
 
 web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')
+
+import zoe_web.web.start
+import zoe_web.web.status

zoe_web/web/start.py

@@ -2,7 +2,7 @@ from flask import render_template
 
 from zoe_client import get_zoe_client
 from zoe_web.web import web_bp
-import zoe_web.web.utils as web_utils
+import zoe_web.utils as web_utils
 
 
 @web_bp.route('/')

zoe_web/web/status.py

+from flask import render_template
+
+from zoe_client import get_zoe_client
+from zoe_web.web import web_bp
+import zoe_web.utils as web_utils
+
+
+@web_bp.route('/status/platform')
+def status_platform():
+    client = get_zoe_client()
+    user = web_utils.check_user(client)
+    template_vars = {
+        "user_id": user.id,
+        "email": user.email
+    }
+    return render_template('status.html', **template_vars)

zoe_web/web/templates/base.html

@@ -11,7 +11,7 @@
 <body>
     <script>
         function update_status() {
-            $.getJSON("{{ url_for("api_status") }}")
+            $.getJSON("{{ url_for("api.status_basic") }}")
                 .done(function( data ) {
                     $("#num_nodes").text(data.num_nodes);
                     $("#num_containers").text(data.num_containers);
@@ -29,7 +29,7 @@
         {% block footer %}
             <div class="status_line">
                 <p>
-                    <span id="app_name"><a href="{{ url_for("web_status") }}">Zoe</a></span>
+                    <span id="app_name"><a href="{{ url_for("web.status_platform") }}">Zoe</a></span>
                     <span id="status">
                         [<span id="num_nodes">N/A</span> swarm nodes, <span id="num_containers">N/A</span> active containers]
                     </span>

zoectl.py

@@ -30,7 +30,7 @@ def user_new_cmd(args):
 def user_get_cmd(args):
     client = get_zoe_client()
     user = client.user_get(args.email)
-    print("User ID: {}".format(user.email))
+    print("User ID: {}".format(user.id))
 
 
 def spark_cluster_new_cmd(args):
@@ -216,7 +216,4 @@ def main():
     args.func(args)
 
 if __name__ == "__main__":
-    try:
-        main()
-    except AttributeError:
-        argparser.print_help()
+    main()