Commit c102b04e authored by Daniele Venzano's avatar Daniele Venzano

Testing ideas for a templating system on top of the current app description

parent bace7012
......@@ -33,6 +33,7 @@ ZApps will be distributed as self-contained set of files with this structure:
* app.json : ZApp description (the current one with a few changes)
* icon.png : an icon to be shown on graphical interfaces
* metadata.json : metadata for the app and options that can be set to modify the app behaviour
* README.md : documentation on the ZApp and its parameters, in markdown format
* docker/
* docker/image1/Dockerfile : the dockerfile needed to build the image for service 1
* docker/image1/* : all the other files required to build the image for service 1
......
......@@ -8,3 +8,4 @@ psycopg2>=2.6.1
pyzmq>=15.2.0
typing
python-consul
jsonschema>=2.5.1
{
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {
"service-options": {
"type": "object",
"properties": {
"instances": {
"type": "object",
"properties": {
"essential_min": {
"type": "integer"
},
"essential_max": {
"type": "integer"
},
"elastic_min": {
"type": "integer"
},
"elastic_max": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"essential_min",
"essential_max",
"elastic_min",
"elastic_max"
]
},
"resources": {
"type": "object",
"properties": {
"memory": {
"type": "object",
"properties": {
"min": {
"type": "integer"
},
"max": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"min",
"max"
]
},
"cores": {
"type": "object",
"properties": {
"min": {
"type": "integer"
},
"max": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"min",
"max"
]
}
},
"additionalProperties": false,
"required": [
"memory",
"cores"
]
},
"parameters": {
"type": "array",
"items": {
"$ref": "#/definitions/parameter"
},
"uniqueItems": true
}
},
"additionalProperties": false,
"required": ["instances", "resources", "parameters"]
},
"parameter": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"environment"
]
},
"type": {
"type": "string",
"enum": [
"int",
"string"
]
},
"name": {
"type": "string"
},
"readable_name": {
"type": "string"
},
"description": {
"type": "string"
},
"default": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"min": { "type": "integer" },
"max": {
"type": "integer"
}
},
"required": ["kind", "type", "name", "readable_name", "description", "default"]
}
},
"type": "object",
"properties": {
"name": {"type": "string"},
"version": {"type": "string"},
"package_version": {
"type": "integer",
"enum": [1]
},
"maintainer": {"type": "string"},
"license": {"type": "string"},
"options": {
"type": "object",
"patternProperties": {
"[a-z\\-]": { "$ref": "#/definitions/service-options" }
},
"minItems": 1
}
},
"additionalProperties": false,
"required": ["name", "version", "package_version", "maintainer", "license"]
}
#!/usr/bin/python3
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Manage ZApp packages.
"""
import logging
from zoe_lib.zapps.zapp_storage import ZAppStorage
from zoe_lib.zapps.zapp_package import ZAppPackage
import zoe_lib.config as config
def list_zapps():
"""List all ZApps available in the local store."""
zapp_store = ZAppStorage()
zapps = zapp_store.list()
for zapp in zapps: # type: ZAppPackage
print('{} {} {} (by {})'.format(zapp.id, zapp.name, zapp.version, zapp.maintainer))
def load_zapp(id):
"""Loads a specific ZApp given its ID."""
zapp_store = ZAppStorage()
zapp = zapp_store.load(id)
if zapp is None:
print('Cannot find the {} ZApp'.format(id))
return
tmp = zapp.generate_app_description()
import pprint; pprint.pprint(tmp)
def main():
"""Main entrypoint."""
config.load_configuration()
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
load_zapp('sleeper:1.0')
if __name__ == '__main__':
main()
......@@ -23,6 +23,7 @@ import zoe_api.master_api
import zoe_lib.applications
import zoe_lib.exceptions
import zoe_lib.state
import zoe_lib.zapps.zapp_storage
from zoe_lib.config import get_conf
log = logging.getLogger(__name__)
......
......@@ -38,6 +38,9 @@ def _check_configuration_sanity():
if config.get_conf().auth_type == 'ldap' and not zoe_api.auth.ldap.LDAP_AVAILABLE:
log.error("LDAP authentication requested, but 'pyldap' module not installed.")
return 1
if not os.access(config.get_conf().zapp_storage, os.W_OK):
log.error("Cannot write to ZApp storage path {}".format(config.get_conf().zapp_storage))
return 1
return 0
......
......@@ -62,17 +62,21 @@ def load_configuration(test_conf=None):
argparser.add_argument('--influxdb-dbname', help='Name of the InfluxDB database to use for storing metrics', default='zoe')
argparser.add_argument('--influxdb-url', help='URL of the InfluxDB service (ex. http://localhost:8086)', default='http://localhost:8086')
argparser.add_argument('--influxdb-enable', action="store_true", help='Enable metric output toward influxDB')
argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:1234)', default='')
argparser.add_argument('--workspace-base-path', help='Path where user workspaces will be created by Zoe. Must be visible at this path on all Swarm hosts.', default='/mnt/zoe-workspaces')
argparser.add_argument('--workspace-deployment-path', help='Path appended to the workspace path to distinguish this deployment. If unspecified is equal to the deployment name.', default='--default--')
argparser.add_argument('--logs-base-path', help='Base path where containers will be able to save logs. Must be a shared directory, visible at this path on all Swarm hosts and writable by Zoe.', default='/mnt/zoe-logs')
argparser.add_argument('--overlay-network-name', help='Name of the Swarm overlay network Zoe should use', default='zoe')
# API options
argparser.add_argument('--listen-address', type=str, help='Address to listen to for incoming connections', default="0.0.0.0")
argparser.add_argument('--listen-port', type=int, help='Port to listen to for incoming connections', default=5001)
argparser.add_argument('--master-url', help='URL of the Zoe master process', default='tcp://127.0.0.1:4850')
argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
# ZApp frontend options
argparser.add_argument('--zapp-storage', help='Path for storing available ZApps', default='/var/lib/zoe')
argparser.add_argument('--registry', help='Docker registry address with port number (ex. 127.0.0.1:5000), leave default to use the Docker Hub', default='')
argparser.add_argument('--registry-repository', help='Organization/Repository used to store ZApp Docker images', default='zapps')
# API auth options
argparser.add_argument('--auth-type', help='Authentication type (text or ldap)', default='text')
......@@ -85,16 +89,18 @@ def load_configuration(test_conf=None):
argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
# Scheduler options
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', choices=['ZoeSimpleScheduler', 'ZoeElasticScheduler'], default='ZoeSimpleScheduler')
argparser.add_argument('--scheduler-policy', help='Scheduler policy to use for scheduling ZApps', choices=['FIFO', 'SIZE'], default='FIFO')
# Backend options
argparser.add_argument('--backend', choices=['OldSwarm', 'OldSwarmNewAPI'], default='OldSwarmNewAPI')
# Docker Swarm backend options
argparser.add_argument('--backend-swarm-url', help='Swarm/Docker API endpoint (ex.: zk://zk1:2181,zk2:2181 or http://swarm:2380)', default='http://localhost:2375')
argparser.add_argument('--backend-swarm-zk-path', help='Swarm/Docker optional ZooKeeper path for Swarm Znodes', default='/docker')
argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
argparser.add_argument('--overlay-network-name', help='Name of the Swarm overlay network Zoe should use', default='zoe')
argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:1234)', default='')
opts = argparser.parse_args()
if opts.debug:
......
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The ZApp packaging definition."""
import json
import os
from jsonschema import validate as json_schema_validate, ValidationError
from zoe_api.exceptions import ZoeException
from zoe_lib.config import get_conf
PACKAGE_VERSION = 1
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
METADATA_SCHEMA_PATH = os.path.realpath(os.path.join(__location__, "..", "..", "schemas", "zapp_metadata_schema.json"))
class ZAppPackage:
"""A ZApp package."""
METADATA_FILE = 'metadata.json'
TEMPLATE_FILE = 'services.json'
DOCS_FILE = 'README.md'
def __init__(self, load_path):
self.load_path = load_path
self.id = ''
# Metadata
self.name = ''
self.version = ''
self.package_version = 0
self.maintainer = ''
self.license = ''
self.sanity_check()
self._load_metadata()
def sanity_check(self):
"""Check that a ZApp package is in the right format."""
if not os.access(os.path.join(self.load_path, self.METADATA_FILE), os.R_OK):
raise ZoeException('No metadata file in ZApp package at {}'.format(self.load_path))
if not os.access(os.path.join(self.load_path, self.DOCS_FILE), os.R_OK):
raise ZoeException('No readme file in ZApp package at {}'.format(self.load_path))
if not os.access(os.path.join(self.load_path, self.TEMPLATE_FILE), os.R_OK):
raise ZoeException('No template file in ZApp package at {}'.format(self.load_path))
def _load_metadata(self):
fpath = os.path.join(self.load_path, self.METADATA_FILE)
schema = json.load(open(METADATA_SCHEMA_PATH, 'r'))
metadata = json.load(open(fpath, 'r'))
json_schema_validate(metadata, schema)
# Code for handling previous package versions goes here
self.package_version = metadata['package_version']
self.name = metadata['name']
self.version = metadata['version']
self.maintainer = metadata['maintainer']
self.license = metadata['license']
self.id = self.name + ':' + self.version
def _load_template(self):
fpath = os.path.join(self.load_path, self.TEMPLATE_FILE)
return json.load(open(fpath, 'r'))
def generate_app_description(self):
"""Generate a Zoe application description by instantiating the template."""
template = self._load_template()
app_descr = {
'name': self.name,
'services': [],
'version': 3,
}
for service in template['services']:
service['image'] = get_conf().registry + '/' + get_conf().registry_repository + '/' + service['image'] + ":" + self.version
app_descr['services'].append(service)
return app_descr
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The ZApp storage manager. It adds, lists, loads and deletes ZApps from the internal store."""
import os
from zoe_lib.config import get_conf
from zoe_lib.exceptions import ZoeLibException
from zoe_lib.zapps.zapp_package import ZAppPackage
ZAPP_STORAGE_PATH = 'zapps'
class ZAppStorage:
"""The ZApp storage manager. It adds, lists, loads and deletes ZApps from the internal store."""
def __init__(self):
self.path = os.path.join(get_conf().zapp_storage, ZAPP_STORAGE_PATH)
try:
os.makedirs(self.path, exist_ok=True)
except OSError:
raise ZoeLibException(message='Cannot create directory {}'.format(self.path))
def list(self):
"""Loads all ZApps from the storage directory and returns a list of ZAppPackage objects."""
all_zapps = []
for zapp_dir in os.listdir(self.path):
zapp = ZAppPackage(os.path.join(self.path, zapp_dir))
all_zapps.append(zapp)
return all_zapps
def load(self, id):
"""Loads a specific ZApp given its ID."""
zapps = self.list()
for zapp in zapps:
if zapp.id == id:
return zapp
return None
......@@ -24,6 +24,8 @@ from zoe_master.workspace.filesystem import ZoeFSWorkspace
log = logging.getLogger(__name__)
LOGS_MOUNT_POINT = '/logs'
def gen_environment(service: Service, execution: Execution):
"""Return the list of environment variables that needs to be added to all containers."""
......@@ -37,7 +39,8 @@ def gen_environment(service: Service, execution: Execution):
('ZOE_OWNER', execution.user_id),
('ZOE_DEPLOYMENT_NAME', get_conf().deployment_name),
('ZOE_MY_DNS_NAME', service.dns_name),
('ZOE_WORKSPACE', fswk.get_mountpoint())
('ZOE_WORKSPACE', fswk.get_mountpoint()),
('ZOE_LOG_STORAGE', LOGS_MOUNT_POINT)
]
service_list = []
for tmp_service in execution.services:
......@@ -67,8 +70,7 @@ def gen_volumes(service: Service, execution: Execution):
logs_path = _create_logs_directories(execution.id, service.name)
if logs_path is not None:
logs_mountpoint = '/logs'
logs_vol = VolumeDescription((logs_path, logs_mountpoint, True))
logs_vol = VolumeDescription((logs_path, LOGS_MOUNT_POINT, True))
vol_list.append(logs_vol)
return vol_list
......
......@@ -38,10 +38,6 @@ class ServiceInstance:
'zoe.deployment_name': get_conf().deployment_name,
'zoe.type': 'app_service'
}
if service.is_monitor:
self.labels['zoe_monitor'] = 'true'
else:
self.labels['zoe_monitor'] = 'false'
self.labels = zoe_master.backends.common.gen_labels(service, execution)
self.environment = service.environment + zoe_master.backends.common.gen_environment(service, execution)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment