Commit 3ef5878e authored by Daniele Venzano's avatar Daniele Venzano

Use JSON schema to validate ZApp descriptions

Update also the port description
parent c4a41385
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
"size": 512, "size": 512,
"services": [ "services": [
{ {
"docker_image": "docker-registry:5000/zapps/spark2-master", "image": "docker-registry:5000/zapps/spark2-master",
"environment": [ "environment": [
[ [
"SPARK_MASTER_IP", "SPARK_MASTER_IP",
...@@ -23,12 +23,10 @@ ...@@ -23,12 +23,10 @@
"name": "spark-master", "name": "spark-master",
"ports": [ "ports": [
{ {
"expose": true,
"is_main_endpoint": true,
"name": "Spark master web interface", "name": "Spark master web interface",
"path": "/", "url_template": "http://{ip_port}/",
"port_number": 8080, "port_number": 8080,
"protocol": "http" "protocol": "tcp"
} }
], ],
"required_resources": { "required_resources": {
...@@ -38,7 +36,7 @@ ...@@ -38,7 +36,7 @@
"total_count": 1 "total_count": 1
}, },
{ {
"docker_image": "docker-registry:5000/zapps/spark2-worker", "image": "docker-registry:5000/zapps/spark2-worker",
"environment": [ "environment": [
[ [
"SPARK_WORKER_CORES", "SPARK_WORKER_CORES",
...@@ -70,11 +68,10 @@ ...@@ -70,11 +68,10 @@
"name": "spark-worker", "name": "spark-worker",
"ports": [ "ports": [
{ {
"is_main_endpoint": false,
"name": "Spark worker web interface", "name": "Spark worker web interface",
"path": "/", "url_template": "http://{ip_port}/",
"port_number": 8081, "port_number": 8081,
"protocol": "http" "protocol": "tcp"
} }
], ],
"required_resources": { "required_resources": {
...@@ -84,7 +81,7 @@ ...@@ -84,7 +81,7 @@
"total_count": 2 "total_count": 2
}, },
{ {
"docker_image": "docker-registry:5000/zapps/spark2-jupyter-notebook", "image": "docker-registry:5000/zapps/spark2-jupyter-notebook",
"environment": [ "environment": [
[ [
"SPARK_MASTER", "SPARK_MASTER",
...@@ -120,19 +117,16 @@ ...@@ -120,19 +117,16 @@
"name": "spark-jupyter", "name": "spark-jupyter",
"ports": [ "ports": [
{ {
"is_main_endpoint": false,
"name": "Spark application web interface", "name": "Spark application web interface",
"path": "/", "url_template": "http://{ip_port}/",
"port_number": 4040, "port_number": 4040,
"protocol": "http" "protocol": "tcp"
}, },
{ {
"expose": true,
"is_main_endpoint": true,
"name": "Jupyter Notebook interface", "name": "Jupyter Notebook interface",
"path": "/", "url_template": "http://{ip_port}/",
"port_number": 8888, "port_number": 8888,
"protocol": "http" "protocol": "tcp"
} }
], ],
"required_resources": { "required_resources": {
...@@ -142,6 +136,6 @@ ...@@ -142,6 +136,6 @@
"total_count": 1 "total_count": 1
} }
], ],
"version": 2, "version": 3,
"will_end": false "will_end": false
} }
...@@ -9,6 +9,8 @@ A Zoe application description is a JSON document. Currently we generate them via ...@@ -9,6 +9,8 @@ A Zoe application description is a JSON document. Currently we generate them via
At the top level map there are some settings, mostly metadata, and a list of services. Each service has its own metadata and some docker-related parameters. At the top level map there are some settings, mostly metadata, and a list of services. Each service has its own metadata and some docker-related parameters.
All fields are required.
Top level Top level
--------- ---------
...@@ -17,42 +19,35 @@ A ZApp is completely contained in a JSON Object. ...@@ -17,42 +19,35 @@ A ZApp is completely contained in a JSON Object.
name name
^^^^ ^^^^
required, string string
The name of this Zapp. Do not confuse this with the name of the execution: you can have many executions (experiment-1, experiment-2) of the same ZApp. The name of this Zapp. Do not confuse this with the name of the execution: you can have many executions (experiment-1, experiment-2) of the same ZApp.
version version
^^^^^^^ ^^^^^^^
required, number number
The ZApp format version of this description. Zoe will check this value before trying to parse the rest of the ZApp to make sure it is able to correctly interpret the description. The ZApp format version of this description. Zoe will check this value before trying to parse the rest of the ZApp to make sure it is able to correctly interpret the description.
will_end will_end
^^^^^^^^ ^^^^^^^^
required, boolean boolean
Must be set to False if potentially this application could run forever. For example a Jupyter notebook will never end (must be terminated explicitly by the user), so needs to have this value set to ``false``. A Spark job instead will finish by itself, so for batch ZApps set this value to ``true``. Must be set to False if potentially this application could run forever. For example a Jupyter notebook will never end (must be terminated explicitly by the user), so needs to have this value set to ``false``. A Spark job instead will finish by itself, so for batch ZApps set this value to ``true``.
size size
^^^^ ^^^^
required, number >= 0 number >= 0
This value is used by the Elastic scheduler as an hint to the application size. This value is used by the Elastic scheduler as an hint to the application size.
disable_autorestart
^^^^^^^^^^^^^^^^^^^
optional, boolean
If set to true, disables all kinds of auorestart on all services of this ZApp.
services services
^^^^^^^^ ^^^^^^^^
required, array array
The list of services to include in this ZApp. The list of services to include in this ZApp.
...@@ -64,14 +59,14 @@ Each service is a JSON Object. At least one service needs to have the monitor ke ...@@ -64,14 +59,14 @@ Each service is a JSON Object. At least one service needs to have the monitor ke
name name
^^^^ ^^^^
required, string string
The name of this service. This value will be combined with other information to generate the unique network names that can be used by services to talk to each other. The name of this service. This value will be combined with other information to generate the unique network names that can be used by services to talk to each other.
environment environment
^^^^^^^^^^^ ^^^^^^^^^^^
required, array array
Environment variables to be passed to the service/container. Each entry in the array must be an array with two elements, the variable name and its value. Environment variables to be passed to the service/container. Each entry in the array must be an array with two elements, the variable name and its value.
...@@ -87,7 +82,7 @@ A number of special values can be used, these will be substituted by Zoe when th ...@@ -87,7 +82,7 @@ A number of special values can be used, these will be substituted by Zoe when th
volumes volumes
^^^^^^^ ^^^^^^^
optional, array array
A list of additional volumes to be mounted in this service container. Each volume is described by an array with three elements: A list of additional volumes to be mounted in this service container. Each volume is described by an array with three elements:
...@@ -97,17 +92,17 @@ A list of additional volumes to be mounted in this service container. Each volum ...@@ -97,17 +92,17 @@ A list of additional volumes to be mounted in this service container. Each volum
Zoe will always mount the user workspace directory in ``$ZOE_WORKSPACE``. Zoe will always mount the user workspace directory in ``$ZOE_WORKSPACE``.
docker_image image
^^^^^^^^^^^^ ^^^^^
required, string string
The full name of the Docker image for this service. The registry can be local, but also images on the Docker Hub will work as expected. The full name of the Docker image for this service. The registry can be local, but also images on the Docker Hub will work as expected.
monitor monitor
^^^^^^^ ^^^^^^^
required, boolean boolean
If set to ``true``, Zoe will monitor this service for termination. When it terminates, Zoe will proceed killing all the other services of the same execution and set the execution status to ``termianted``. If set to ``true``, Zoe will monitor this service for termination. When it terminates, Zoe will proceed killing all the other services of the same execution and set the execution status to ``termianted``.
If set to ``false``, Zoe will configure Docker to automatically restart the service in case it crashes. If set to ``false``, Zoe will configure Docker to automatically restart the service in case it crashes.
...@@ -119,82 +114,70 @@ All autorestart behaviour is disabled if the global parameter ``disable_autorest ...@@ -119,82 +114,70 @@ All autorestart behaviour is disabled if the global parameter ``disable_autorest
total_count total_count
^^^^^^^^^^^ ^^^^^^^^^^^
required, number number
The maximum number of services of this type (with the same docker image and associated options) that can be started by Zoe. The maximum number of services of this type (with the same docker image and associated options) that can be started by Zoe.
essential_count essential_count
^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^
required, number <= total_count number <= total_count
The minimum number of services of this type that Zoe must start before being able to consider the ZApp as started. For example, in Spark you need just one worker to produce useful work (essential_count equal to 1), but if there is the possibility of adding up to 9 more workers, the application will run faster (total_count equal to 10). The minimum number of services of this type that Zoe must start before being able to consider the ZApp as started. For example, in Spark you need just one worker to produce useful work (essential_count equal to 1), but if there is the possibility of adding up to 9 more workers, the application will run faster (total_count equal to 10).
required_resources required_resources
^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^
required, object object
Resources that need to be reserved for this service. Currently only ``memory`` is supported, specified in bytes. Resources that need to be reserved for this service. Currently only ``memory`` is supported, specified in bytes.
startup_order startup_order
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
required, number number
Relative ordering for service startup. Zoe will start first services with a lower value. Note that Zoe will not wait for the service to be up and running before starting the next in the list. Relative ordering for service startup. Zoe will start first services with a lower value. Note that Zoe will not wait for the service to be up and running before starting the next in the list.
ports ports
^^^^^ ^^^^^
required, array array
A list of ports that the user may wants to access. Currently this is tailored for web interfaces, URLs for each port will be shown in the client interfaces. See the *port* section below for details. A list of ports that the user may wants to access. Currently this is tailored for web interfaces, URLs for each port will be shown in the client interfaces. See the *port* section below for details.
Ports Ports
----- -----
Zoe will instruct the backend to expose ports on public addresses. This is usually done by port forwarding and depends on the capabilities of the configured back-end.
name name
^^^^ ^^^^
required, string string
A user friendly description for the service exposed on this port. A user friendly description for the service exposed on this port.
path url_template
^^^^ ^^^^^^^^^^^^
optional, string string
The path part of the URL, after the port number. Must start with '/'. A template for the full URL that will be exposed to the user. Zoe will query the backend at run time to get the public IP address and port combination and substitute the ``{ip_port}`` part.
protocol protocol
^^^^^^^^ ^^^^^^^^
required, string string
The URL protocol
is_main_endpoint
^^^^^^^^^^^^^^^^
required, boolean
Used to emphasize certain service endpoints in the user interface.
expose
^^^^^^
optional, boolean
Expose this port on a public IP address vie Docker. This feature in incomplete: it works only on TCP port and Zoe will not show anywhere the public IP address, that will be available only by using Docker tools. The protocol, either ``tcp`` or ``udp``.
port_number port_number
^^^^^^^^^^^ ^^^^^^^^^^^
required, number number
The port number where this service endpoint is exposed. The port number where the service is listening for connections. The external (user-visible) port number will be chosen by the back-end.
Example Example
------- -------
...@@ -202,7 +185,7 @@ Example ...@@ -202,7 +185,7 @@ Example
{ {
"name": "Jupyter notebook", "name": "Jupyter notebook",
"version": 2, "version": 3,
"will_end": false, "will_end": false,
"size": 512, "size": 512,
"services": [ "services": [
...@@ -211,7 +194,7 @@ Example ...@@ -211,7 +194,7 @@ Example
"environment": [ "environment": [
["NB_USER", "{user_name}"] ["NB_USER", "{user_name}"]
], ],
"docker_image": "docker-registry:5000/apps/jupyter-notebook", "image": "docker-registry:5000/apps/jupyter-notebook",
"monitor": true, "monitor": true,
"total_count": 1, "total_count": 1,
"essential_count": 1, "essential_count": 1,
...@@ -222,10 +205,8 @@ Example ...@@ -222,10 +205,8 @@ Example
"ports": [ "ports": [
{ {
"name": "Jupyter Notebook interface", "name": "Jupyter Notebook interface",
"path": "/", "url_template": "http://{ip_port}/",
"protocol": "http", "protocol": "tcp",
"is_main_endpoint": true,
"expose": true,
"port_number": 8888 "port_number": 8888
} }
] ]
......
...@@ -11,3 +11,4 @@ python-oauth2 ...@@ -11,3 +11,4 @@ python-oauth2
python-consul python-consul
pykube>=0.14.0 pykube>=0.14.0
sphinx sphinx
jsonschema
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Zoe application description schema",
"definitions": {
"service": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 3,
"maxLength": 16,
"pattern": "^[a-zA-Z0-9\\-]*$"
},
"image": {
"type": "string",
"minLength": 1
},
"ports": {
"type": "array",
"items": {
"$ref": "#/definitions/port"
},
"minItems": 0,
"uniqueItems": true
},
"environment": {
"type": "array",
"items": {
"$ref": "#/definitions/env_var"
},
"minItems": 0,
"uniqueItems": true
},
"volumes": {
"type": "array",
"items": {
"$ref": "#/definitions/volume"
},
"minItems": 0,
"uniqueItems": true
},
"monitor": {
"type": "boolean"
},
"startup_order": {
"type": "number",
"multipleOf": 1.0
},
"essential_count": {
"type": "number",
"multipleOf": 1.0,
"minimum": 1
},
"total_count": {
"type": "number",
"multipleOf": 1.0,
"minimum": 1
},
"resources": {
"type": "object",
"properties": {
"memory": {
"type": "object",
"properties": {
"min": {
"oneOf": [
{
"type": "number",
"multipleOf": 1.0,
"minimum": 1024
},
{
"type": "null"
}
]
},
"max": {
"oneOf": [
{
"type": "number",
"multipleOf": 1.0
},
{
"type": "null"
}
]
}
},
"additionalProperties": false,
"required": [
"min",
"max"
]
},
"cores": {
"type": "object",
"properties": {
"min": {
"oneOf": [
{
"type": "number",
"minimum": 0,
"exclusiveMinimum": true
},
{
"type": "null"
}
]
},
"max": {
"oneOf": [
{
"type": "number"
},
{
"type": "null"
}
]
}
},
"additionalProperties": false,
"required": [
"min",
"max"
]
}
},
"additionalProperties": false,
"required": [
"memory",
"cores"
]
},
"command": {
"oneOf": [
{ "type": "string", "minLength": 1 },
{ "type": "null" }
]
}
},
"additionalProperties": false,
"required": ["name", "image", "ports", "environment", "volumes", "monitor", "startup_order", "essential_count", "total_count", "resources", "command"]
},
"port": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 3
},
"port_number": {
"type": "number",
"multipleOf": 1.0,
"minimum": 0,
"exclusiveMinimum": true,
"maximum": 65536,
"exclusiveMaximum": true
},
"protocol": {
"type": "string",
"enum": ["tcp", "udp"]
},
"url_template": {
"type": "string",
"regex": "^[a-z]?://HOST:PORT/[a-zA-Z0-9./\\-_]$"
}
},
"additionalProperties": false,
"required": ["name", "port_number", "protocol", "url_template"]
},
"env_var": {
"type": "array",
"items": [
{
"type": "string",
"regex": "^[A-Z][A-Z0-9]*$"
},
{
"type": "string"
}
]
},
"volume": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 3,
"regex": "^[a-zA-Z0-9.]?$"
},
"path": {
"type": "string",
"minLength": 2,
"regex": "^/[a-zA-Z0-9]?[a-zA-Z0-9./]*$"
},
"read_only": {
"type": "boolean"
}
}
}
},
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 3,
"maxLength": 16,
"pattern": "^[a-zA-Z0-9\\-]*$"
},
"will_end": {
"type": "boolean"
},
"size": {
"type": "integer",
"minimum": 0
},
"version": {
"type": "number",
"multipleOf": 1.0,
"minimum": 3
},
"services": {
"type": "array",
"items": {
"$ref": "#/definitions/service"
},
"minItems": 1,
"uniqueItems": true
}
},
"additionalProperties": false,
"required": ["name", "will_end", "size", "version", "services"]
}
...@@ -172,3 +172,18 @@ class APIEndpoint: ...@@ -172,3 +172,18 @@ class APIEndpoint:
self.master.execution_terminate(execution.id) self.master.execution_terminate(execution.id)
break break
log.debug('Cleanup task finished') log.debug('Cleanup task finished')
def execution_endpoints(self, uid: str, role: str, execution: zoe_lib.state.Execution):
"""Return a list of the services and public endpoints available for a certain execution."""
services_info = []
endpoints = []
for service in execution.services:
services_info.append(self.service_by_id(uid, role, service.id))
port_mappings = service.ports
for port in service.description['ports']:
port_number = str(port['port_number']) + "/" + port['protocol']
if port_number in port_mappings:
endpoint = port['url_template'].format(**{"ip_port": port_mappings[port_number][0] + ":" + port_mappings[port_number][1]})
endpoints.append((port['name'], endpoint))
return services_info, endpoints
...@@ -19,7 +19,7 @@ from typing import List ...@@ -19,7 +19,7 @@ from typing import List
import tornado.web import tornado.web
from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI, ExecutionDeleteAPI from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI, ExecutionDeleteAPI, ExecutionEndpointsAPI
from zoe_api.rest_api.info import InfoAPI from zoe_api.rest_api.info import InfoAPI
from zoe_api.rest_api.userinfo import UserInfoAPI from zoe_api.rest_api.userinfo import UserInfoAPI
from zoe_api.rest_api.service import ServiceAPI from zoe_api.rest_api.service import ServiceAPI
...@@ -46,6 +46,7 @@ def api_init(api_endpoint) -> List[tornado.web.URLSpec]: ...@@ -46,6 +46,7 @@ def api_init(api_endpoint) -> List[tornado.web.URLSpec]:
tornado.web.url(API_PATH + r'/execution/([0-9]+)', ExecutionAPI, route_args), tornado.web.url(API_PATH + r'/execution/([0-9]+)', ExecutionAPI, route_args),
tornado.web.url(API_PATH + r'/execution/delete/([0-9]+)', ExecutionDeleteAPI, route_args), tornado.web.url(API_PATH + r'/execution/delete/([0-9]+)', ExecutionDeleteAPI, route_args),
tornado.web.url(API_PATH + r'/execution/endpoints/([0-9]+)', ExecutionEndpointsAPI, route_args),
tornado.web.url(API_PATH + r'/execution', ExecutionCollectionAPI, route_args), tornado.web.url(API_PATH + r'/execution', ExecutionCollectionAPI, route_args),
tornado.web.url(API_PATH + r'/service/([0-9]+)', ServiceAPI, route_args), tornado.web.url(API_PATH + r'/service/([0-9]+)', ServiceAPI, route_args),
......
...@@ -117,7 +117,7 @@ class ExecutionCollectionAPI(RequestHandler): ...@@ -117,7 +117,7 @@ class ExecutionCollectionAPI(RequestHandler):
manage_cors_headers(self) manage_cors_headers(self)