Merge branch 'devel/platform_status' into 'master'

Allow resource limits to be set by the users on the web interface

See merge request !42

zoe_api/web/__init__.py

@@ -50,7 +50,7 @@ def web_init(api_endpoint) -> List[tornado.web.URLSpec]:
         tornado.web.url(r'/websocket', zoe_api.web.websockets.WebSocketEndpointWeb, route_args, name='websocket'),
 
         tornado.web.url(r'/zapp-shop', zoe_api.web.zapp_shop.ZAppShopHomeWeb, route_args, name='zappshop'),
-        tornado.web.url(r'/zapp-shop/logo/([a-z\-.]+)', zoe_api.web.zapp_shop.ZAppLogoWeb, route_args, name='zappshop_logo'),
+        tornado.web.url(r'/zapp-shop/logo/([0-9a-z\-.]+)', zoe_api.web.zapp_shop.ZAppLogoWeb, route_args, name='zappshop_logo'),
         tornado.web.url(r'/zapp-shop/start/([0-9a-z\-.]+)', zoe_api.web.zapp_shop.ZAppStartWeb, route_args, name='zappshop_start'),
 
         tornado.web.url(r'/status', zoe_api.web.status.StatusEndpointWeb, route_args, name='status')

zoe_api/web/static/zoe.css

@@ -180,6 +180,27 @@ form#zapp_start_form input {
     font-family: "Lucida Console", Monaco, monospace;
 }
 
+form#zapp_start_form label.label-inline,
+form#zapp_start_form label.label-inline input {
+    font-variant: normal;
+    display: inline;
+}
+
+.resource-form label {
+    float: left;
+    padding-bottom: 1.5em;
+    padding-right: 2em;
+
+}
+
+form#zapp_start_form h5,
+form#zapp_start_form hr {
+    clear: both;
+    border: 0;
+    padding-bottom: 0;
+    margin-bottom: 10px;
+}
+
 /* footer */
 
 div.status_line {

zoe_api/web/static/zoe.js

@@ -3,9 +3,9 @@ function format_bytes(bytes, decimals) {
         document.write('0 Byte');
         return;
     }
-    var k = 1000;
+    var k = 1024;
     var dm = decimals + 1 || 3;
-    var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    var sizes = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
     var i = Math.floor(Math.log(bytes) / Math.log(k));
     document.write((bytes / Math.pow(k, i)).toPrecision(dm) + ' ' + sizes[i]);
 }
@@ -14,9 +14,9 @@ function format_bytes_ret(bytes, decimals) {
     if(bytes === 0) {
         return '0 Byte';
     }
-    var k = 1000;
+    var k = 1024;
     var dm = decimals + 1 || 3;
-    var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    var sizes = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
     var i = Math.floor(Math.log(bytes) / Math.log(k));
     return (bytes / Math.pow(k, i)).toPrecision(dm) + ' ' + sizes[i];
 }

zoe_api/web/templates/zapp_shop.html

@@ -12,7 +12,7 @@
         <div class="zapp-list">
             {% for zapp in zapps[zapp_category] %}
             <div class="zapp" id="{{ zapp.id }}-{{ zapp.manifest_index }}">
-                <img src="{{ reverse_url("zappshop_logo", zapp.id) }}" alt="logo">
+                <img src="{{ reverse_url("zappshop_logo", zapp.id + "-" + zapp.manifest_index|string) }}" alt="logo">
                 <p>{{ zapp.readable_name }}</p>
                 <div class="readable_description" id="rd-{{ zapp.id }}">{{ zapp.readable_description|safe }}</div>
             </div>

zoe_api/web/templates/zapp_start.html

@@ -9,7 +9,7 @@
 {% block content %}
 
     <div class="zapp-description">
-        <img src="{{ reverse_url("zappshop_logo", zapp.id) }}" alt="logo">
+        <img src="{{ reverse_url("zappshop_logo", zapp.id + '-' + zapp.manifest_index|string) }}" alt="logo">
         {{ zapp.readable_description|safe }}
     </div>
 
@@ -17,14 +17,13 @@
     <p>This ZApp is composed by the following services:</p>
     <ul>
     {% for service in zapp.zoe_description.services %}
-        <li>{{ service["total_count"] }} {{ service["name"] }} ({{ service["essential_count"] }} essential{{ 's' if service["essential_count"] > 1 }})
-        <ul>
-            <li>memory: {% if service["resources"]["memory"]["max"] != None %}
-                <script>format_bytes({{ service["resources"]["memory"]["max"] }}, 2) </script></li>
+        <li>{{ service["total_count"] }} {{ service["name"] }} ({{ service["essential_count"] }} essential{{ 's' if service["essential_count"] > 1 }})        <ul>
+            <li>Suggested memory allocation: {% if service["resources"]["memory"]["min"] != None %}
+                <script>format_bytes({{ service["resources"]["memory"]["min"] }}, 2) </script>
                         {% else %}
                 No limit
-                        {% endif %}
-            <li>cpu cores: {{ service["resources"]["cores"]["max"] if service["resources"]["cores"]["max"] != None else "No limit" }}</li>
+                        {% endif %}</li>
+            <li>Suggested CPU core allocation limit: {{ service["resources"]["cores"]["min"] if service["resources"]["cores"]["min"] != None else "No limit" }}</li>
         </ul></li>
     {% endfor %}
     </ul>
@@ -34,14 +33,54 @@
     <input type="hidden" name="zapp-id" value="{{ zapp.id }}-{{ zapp.manifest_index }}">
     <label>Execution name:&nbsp;<input type="text" name="exec_name" value="{{ zapp.zoe_description.name }}" maxlength="16" size="18" required/></label><br/>
     {% for param in zapp.parameters %}
-        <label>{{ param.readable_name }}:&nbsp;
-        <input name="{{ param.name }}" value="{{ param.default if param.default != None }}" required title="{{ param.description }}" size="128"/></label><br/>
+        <label>{{ param.readable_name }}
+            {% if param.type == "number" and "memory" in param.kind %}
+        (GiB) <input name="{{ param.name }}-{{ param.kind }}" value="{{ param.default }}" required size="5" title="{{ param.description }}" type="{{ param.type }}" min="{{ param.min }}" max="{{ param.max }}" step="{{ param.step }}"/>
+            {% elif param.type == "number" %}
+        <input name="{{ param.name }}-{{ param.kind }}" value="{{ param.default }}" required size="5" title="{{ param.description }}" type="{{ param.type }}" min="{{ param.min }}" max="{{ param.max }}" step="{{ param.step }}"/>
+            {% else %}
+        <input name="{{ param.name }}-{{ param.kind }}" value="{{ param.default if param.default != None }}" required title="{{ param.description }}" size="128" type="{{ param.type }}" />
+            {% endif %}
+        </label><br/>
     {% endfor %}
-        <button type="submit">Start</button>
+    {% if resources_are_customizable %}
+        {% for service in zapp.zoe_description.services %}
+        <div class="resource-form">
+        <h5>{{ service.name }}</h5>
+        <label>Memory allocation (GiB)
+            <input name="{{ service.name }}-resource_memory_min" value="{{ service["resources"]["memory"]["min"] / (1024 ** 3) }}" required title="Memory soft limit, service can use more memory if there is no contention" type="number" min="0.5" max="{{ max_memory_limit }}" step="0.5"/></label>
+        <label>Minimum core allocation:
+            <input name="{{ service.name }}-resource_cores_min" value="{{ service["resources"]["cores"]["min"] }}" required title="No less than this amount of cores will be allocated to this service" type="number" min="0.1" max="{{ max_core_limit }}" step="0.1"/></label>
+        </div>
+        {%  endfor %}
+    {% endif %}
+    <hr>
+    {%  if role != "guest" %}
+        <label class="label-inline"><input type="checkbox" onchange="set_submit_text(this)" name="download_json"> Download JSON with these parameters for command-line execution</label>
+        <br><br>
+    {% endif %}
+        <button id="submit" type="submit">Start</button>
+        <button type="reset">Reset</button>
     </form>
 
     <script>
         $("#zapp_start_form").validate();
+
+        function set_submit_text(elem) {
+            if (elem.checked) {
+                $("#submit").text("Download JSON");
+            } else {
+                $("#submit").text("Start");
+            }
+        }
+
+        function update_range_value(value, id, bytes) {
+            if (bytes) {
+                $('#' + id).text(format_bytes_ret(value, 2));
+            } else {
+                $('#' + id).text(value);
+            }
+        }
     </script>
 
     <p><a href="{{ reverse_url("zappshop") }}">Back to the ZApp shop</a></p>

zoe_api/web/zapp_shop.py

@@ -21,6 +21,7 @@ from zoe_api import zapp_shop
 from zoe_api.api_endpoint import APIEndpoint  # pylint: disable=unused-import
 from zoe_api.web.utils import get_auth, catch_exceptions
 from zoe_api.web.custom_request_handler import ZoeRequestHandler
+from zoe_lib.config import get_conf
 
 log = logging.getLogger(__name__)
 
@@ -64,7 +65,12 @@ class ZAppLogoWeb(ZoeRequestHandler):
             return self.redirect(self.get_argument('next', u'/login'))
 
         self.set_header("Content-type", "image/png")
-        self.write(zapp_shop.get_logo(zapp_id))
+
+        manifest_index = int(zapp_id.split('-')[-1])
+        zapp_id = "-".join(zapp_id.split('-')[:-1])
+        zapps = zapp_shop.zshop_read_manifest(zapp_id)
+        zapp = zapps[manifest_index]
+        self.write(zapp_shop.get_logo(zapp))
 
 
 class ZAppStartWeb(ZoeRequestHandler):
@@ -89,7 +95,10 @@ class ZAppStartWeb(ZoeRequestHandler):
         template_vars = {
             "uid": uid,
             "role": role,
-            'zapp': zapp
+            'zapp': zapp,
+            'max_core_limit': get_conf().max_core_limit,
+            'max_memory_limit': get_conf().max_memory_limit,
+            'resources_are_customizable': role == "admin" or (role != "guest" and (role == "user" and not get_conf().no_user_edit_limits_web))
         }
         self.render('zapp_start.html', **template_vars)
 
@@ -107,23 +116,50 @@ class ZAppStartWeb(ZoeRequestHandler):
 
         exec_name = self.get_argument('exec_name')
 
-        app_descr = self._set_parameters(zapp.zoe_description, zapp.parameters)
+        app_descr = self._set_parameters(zapp.zoe_description, zapp.parameters, role)
 
-        new_id = self.api_endpoint.execution_start(uid, role, exec_name, app_descr)
+        download_json = self.get_argument('download_json')
+        if download_json:
+            self.set_header('Content-Type', 'application/json')
+            self.set_header('Content-Disposition', 'attachment; filename={}.json'.format(zapp_id))
+            self.write(app_descr)
+            self.finish()
+            return
+        else:
+            new_id = self.api_endpoint.execution_start(uid, role, exec_name, app_descr)
 
         self.redirect(self.reverse_url('execution_inspect', new_id))
 
-    def _set_parameters(self, app_descr, params):
+    def _set_parameters(self, app_descr, params, role):
         for param in params:
+            argument_name = param.name + '-' + param.kind
             if param.kind == 'environment':
                 for service in app_descr['services']:
                     for env in service['environment']:
                         if env[0] == param.name:
-                            env[1] = self.get_argument(param.name)
+                            env[1] = self.get_argument(argument_name)
             elif param.kind == 'command':
                 for service in app_descr['services']:
                     if service['name'] == param.name:
-                        service['command'] = self.get_argument(param.name)
+                        service['command'] = self.get_argument(argument_name)
+                        break
+            elif param.kind == 'resource_memory_min' and role == "admin" or (role != "guest" and (role == "user" and not get_conf().no_user_edit_limits_web)):
+                for service in app_descr['services']:
+                    if service['name'] == param.name:
+                        if self.get_argument(argument_name) >= get_conf().max_memory_limit * (1024 ** 3):
+                            val = get_conf().max_memory_limit * (1024 ** 3)
+                        else:
+                            val = self.get_argument(argument_name)
+                        service["resources"]["memory"]["min"] = val
+                        break
+            elif param.kind == 'resource_cores_min' and role == "admin" or (role != "guest" and (role == "user" and not get_conf().no_user_edit_limits_web)):
+                for service in app_descr['services']:
+                    if service['name'] == param.name:
+                        if self.get_argument(argument_name) >= get_conf().max_core_limit:
+                            val = get_conf().max_core_limit
+                        else:
+                            val = self.get_argument(argument_name)
+                        service["resources"]["cores"]["min"] = val
                         break
             else:
                 log.warning('Unknown parameter kind: {}, ignoring...'.format(param.kind))

zoe_api/zapp_shop.py

@@ -62,6 +62,10 @@ class ZApp:
             self.guest_access = zapp['guest_access']
         else:
             self.guest_access = False
+        if 'logo' in zapp:
+            self.logo = zapp['logo']
+        else:
+            self.logo = 'logo.png'
 
     def parse_parameters(self, zapp_manifest):
         """Translates the parameters from the manifest into objects."""
@@ -110,7 +114,7 @@ def zshop_read_manifest(zapp_id):
     return zapps
 
 
-def get_logo(zapp_id):
+def get_logo(zapp: ZApp):
     """Return the ZApp PNG logo image file contents."""
-    logo_path = os.path.join(get_conf().zapp_shop_path, zapp_id, 'logo.png')
+    logo_path = os.path.join(get_conf().zapp_shop_path, zapp.id, zapp.logo)
     return open(logo_path, "rb").read()

zoe_lib/config.py

@@ -116,6 +116,9 @@ def load_configuration(test_conf=None):
         # other options
         argparser.add_argument('--zapp-shop-path', help='Path where ZApp folders are stored', default='/var/lib/zoe-apps')
         argparser.add_argument('--log-file', help='output logs to a file', default='stderr')
+        argparser.add_argument('--max-core-limit', help='Maximum amount of cores users are able to reserve', type=int, default=16)
+        argparser.add_argument('--max-memory-limit', help='Maximum amount of memory services can use (in GiB)', type=int, default=64)
+        argparser.add_argument('--no-user-edit-limits-web', action='store_true', help='Disable editing ZApp resource limits from the web interface (only admins will able to)')
 
         argparser.add_argument('--aml-ttl', help='TimeToLive in hours for AML executions', type=int, default=4)
 

zoe_master/backends/service_instance.py

@@ -35,11 +35,15 @@ class ServiceInstance:
             self.memory_limit = None
         else:
             self.memory_limit = service.resource_reservation.memory
+            if self.memory_limit.max > get_conf().max_memory_limit * (1024 ** 3):
+                self.memory_limit.max = get_conf().max_memory_limit * (1024 ** 3)
 
         if service.resource_reservation.cores.min is None:
             self.core_limit = None
         else:
             self.core_limit = service.resource_reservation.cores
+            if self.core_limit.max > get_conf().max_core_limit:
+                self.core_limit = get_conf().max_core_limit
 
         self.labels = {
             'zoe.execution.name': execution.name,