Commit 60abe098 authored by Daniele Venzano's avatar Daniele Venzano

Move Zoe application definitions to their own repository

parent 4b3de173
{
"name": "aml-lab",
"priority": 512,
"requires_binary": false,
"services": [
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-master",
"environment": [
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
]
],
"monitor": false,
"name": "spark-master",
"networks": [
"eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792"
],
"ports": [
{
"is_main_endpoint": false,
"name": "Spark master web interface",
"path": "/",
"port_number": 8080,
"protocol": "http"
}
],
"required_resources": {
"memory": 536870912
}
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-worker",
"environment": [
[
"SPARK_WORKER_CORES",
"6"
],
[
"SPARK_WORKER_RAM",
"11273240064"
],
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
],
[
"SPARK_LOCAL_IP",
"spark-worker0-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
]
],
"monitor": false,
"name": "spark-worker0",
"networks": [
"eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792"
],
"ports": [
{
"is_main_endpoint": false,
"name": "Spark worker web interface",
"path": "/",
"port_number": 8081,
"protocol": "http"
}
],
"required_resources": {
"memory": 12884901888
}
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-jupyter-notebook",
"environment": [
[
"SPARK_MASTER",
"spark://spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe:7077"
],
[
"SPARK_EXECUTOR_RAM",
"11273240064"
],
[
"SPARK_DRIVER_RAM",
"2147483648"
],
[
"NB_USER",
"{user_name}"
],
[
"NAMENODE_HOST",
"hdfs-namenode.hdfs"
]
],
"monitor": true,
"name": "spark-jupyter",
"networks": [
"eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792"
],
"ports": [
{
"is_main_endpoint": false,
"name": "Spark application web interface",
"path": "/",
"port_number": 4040,
"protocol": "http"
},
{
"is_main_endpoint": true,
"name": "Jupyter Notebook interface",
"path": "/",
"port_number": 8888,
"protocol": "http"
}
],
"required_resources": {
"memory": 4294967296
}
},
{
"docker_image": "192.168.45.252:5000/zoerepo/spark-worker",
"environment": [
[
"SPARK_WORKER_CORES",
"6"
],
[
"SPARK_WORKER_RAM",
"11273240064"
],
[
"SPARK_MASTER_IP",
"spark-master-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
],
[
"SPARK_LOCAL_IP",
"spark-worker1-{execution_name}-{user_name}-{deployment_name}-zoe.{user_name}-{deployment_name}-zoe"
]
],
"monitor": false,
"name": "spark-worker1",
"networks": [
"eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792"
],
"ports": [
{
"is_main_endpoint": false,
"name": "Spark worker web interface",
"path": "/",
"port_number": 8081,
"protocol": "http"
}
],
"required_resources": {
"memory": 12884901888
}
}
],
"version": 1,
"will_end": false
}
#!/usr/bin/python3
import os
import json
from zoe_lib.predefined_apps import openmpi
from zoe_lib.workflow import ZoeWorkFlow
WORKSPACE_BASE_PATH = '/mnt/cephfs/zoe-workspaces'
ZOE_WORKER_JSON = '''
{
"command": "",
"docker_image": "192.168.45.252:5000/zoerepo/openmpi-worker",
"environment": [],
"monitor": false,
"name": "mpiworker",
"ports": [],
"required_resources": {
"memory": 1073741824
},
"volumes": []
}
'''
ZOE_MPIRUN_JSON = '''
{
"command": "mpirun <options>",
"docker_image": "192.168.45.252:5000/zoerepo/openmpi-worker",
"environment": [],
"monitor": true,
"name": "mpirun",
"ports": [],
"required_resources": {
"memory": 1073741824
},
"volumes": []
}
'''
ZOE_APP_BASE_JSON = '''
{
"name": "openmpi-hello",
"priority": 512,
"requires_binary": true,
"services": [],
"version": 1,
"will_end": true
}
'''
def prepare_mpirun(wf):
count = 4
mpihosts = ''
......@@ -16,7 +58,15 @@ def prepare_mpirun(wf):
mpihosts += '\n'
wf.workspace.put_string(mpihosts, 'mpihosts')
cmdline = 'mpirun -np {} --hostfile mpihosts ./MPI_Hello'.format(count)
zoe_app = openmpi.openmpi_app(name='mpi-hello-world', mpirun_commandline=cmdline, worker_count=count)
zoe_app = json.loads(ZOE_APP_BASE_JSON)
zoe_app['name'] = 'mpi-hello-world'
mpirun_service = json.loads(ZOE_MPIRUN_JSON)
mpirun_service['command'] = cmdline
for wc in range(count):
mpiworker = json.loads(ZOE_WORKER_JSON)
mpiworker['name'] = 'mpiworker{}'.format(wc)
zoe_app['services'].append(mpiworker)
zoe_app['services'].append(mpirun_service)
return zoe_app
......
......@@ -27,10 +27,10 @@ from pprint import pprint
from zoe_cmd import utils
from zoe_lib.users import ZoeUserAPI
from zoe_lib.services import ZoeServiceAPI
from zoe_lib.exceptions import ZoeAPIException
from zoe_lib.exceptions import ZoeAPIException, InvalidApplicationDescription
from zoe_lib.executions import ZoeExecutionsAPI
from zoe_lib.query import ZoeQueryAPI
from zoe_lib.applications import app_validate, predefined_app_generate, predefined_app_list
from zoe_lib.applications import app_validate
def stats_cmd(_):
......@@ -78,19 +78,14 @@ def user_list_cmd(_):
print('Gateway URLs: {}'.format(user['gateway_urls']))
def pre_app_list_cmd(_):
for a in predefined_app_list():
print(a)
def pre_app_export_cmd(args):
def app_validate_cmd(args):
app_descr = json.load(args.jsonfile)
try:
app = predefined_app_generate(args.app_name)
except ZoeAPIException:
print('Application not found')
app_validate(app_descr)
except InvalidApplicationDescription as e:
print(e)
else:
json.dump(app, sys.stdout, sort_keys=True, indent=4)
print()
print("Static validation OK")
def app_get_cmd(args):
......@@ -179,12 +174,9 @@ def process_arguments() -> Namespace:
argparser_user_list = subparser.add_parser('user-ls', help='Lists all users defined in the system')
argparser_user_list.set_defaults(func=user_list_cmd)
argparser_pre_app_list = subparser.add_parser('pre-app-ls', help='Lists the predefined application descriptions')
argparser_pre_app_list.set_defaults(func=pre_app_list_cmd)
argparser_pre_app_export = subparser.add_parser('pre-app-export', help='Export one of the predefined application descriptions in JSON (stdout)')
argparser_pre_app_export.add_argument('app_name', help='Predefined application name (use pre-app-list to see what is available')
argparser_pre_app_export.set_defaults(func=pre_app_export_cmd)
argparser_app_validate = subparser.add_parser('app-validate', help='Validate an application description')
argparser_app_validate.add_argument('jsonfile', type=FileType("r"), help='Application description')
argparser_app_validate.set_defaults(func=app_validate_cmd)
argparser_exec_start = subparser.add_parser('start', help="Start an application")
argparser_exec_start.add_argument('name', help="Name of the execution")
......
......@@ -19,39 +19,11 @@ This module contains code to validate application descriptions.
import logging
from zoe_lib.exceptions import InvalidApplicationDescription, ZoeAPIException
from zoe_lib.predefined_apps import PREDEFINED_APPS
from zoe_lib.exceptions import InvalidApplicationDescription
log = logging.getLogger(__name__)
def predefined_app_list():
"""
Returns a list of predefined application available
:return: a list of application names
"""
name_list = []
for gen_app in PREDEFINED_APPS:
app = gen_app()
name_list.append(app['name'])
return name_list
def predefined_app_generate(name):
"""
Returns the predefined application corresponding to the name given as argument
:param name: the name of the application to generate
:return: an application dictionary
"""
for gen_app in PREDEFINED_APPS:
app = gen_app()
if app['name'] == name:
return app
raise ZoeAPIException('No such predefined application')
def app_validate(data):
"""
Validates an application description, making sure all required fields are present and of the correct type.
......
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_lib.predefined_apps.copier import copier_app
from zoe_lib.predefined_apps.spark_interactive import spark_jupyter_notebook_app
from zoe_lib.predefined_apps.eurecom_aml_lab import spark_jupyter_notebook_lab_app
from zoe_lib.predefined_apps.hdfs import hdfs_app
from zoe_lib.predefined_apps.openmpi import openmpi_app
from zoe_lib.predefined_apps.spark_submit import spark_submit_app
from zoe_lib.predefined_apps.test_sleep import sleeper_app
PREDEFINED_APPS = [
copier_app,
spark_jupyter_notebook_app,
spark_jupyter_notebook_lab_app,
hdfs_app,
openmpi_app,
spark_submit_app,
sleeper_app
]
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_lib.predefined_frameworks.utils import copier_service
empty = {
'host_path': 'CHANGEME', # the path containing what to copy
'cont_path': 'CHANGEME', # the file or directory to copy from or to host_path
'readonly': False
}
def copier_app(src_volume=empty, src_path='', dst_volume=empty, dst_path=''):
"""
:type src_volume: dict
:type src_path: str
:type dst_volume: dict
:type dst_path: str
:rtype: dict
"""
app = {
'name': 'copier',
'version': 1,
'will_end': True,
'priority': 512,
'requires_binary': False,
'services': [
copier_service(src_volume, src_path, dst_volume, dst_path)
]
}
return app
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zoe_lib.predefined_frameworks.jupyter_spark as jupyter_framework
import zoe_lib.predefined_frameworks.spark as spark_framework
def spark_jupyter_notebook_lab_app(name='aml-lab',
notebook_mem_limit=4 * (1024 ** 3),
master_mem_limit=512 * (1024 ** 2),
worker_count=2,
worker_mem_limit=12 * (1024 ** 3),
worker_cores=6,
master_image='192.168.45.252:5000/zoerepo/spark-master',
worker_image='192.168.45.252:5000/zoerepo/spark-worker',
notebook_image='192.168.45.252:5000/zoerepo/spark-jupyter-notebook'):
"""
:type name: str
:type notebook_mem_limit: int
:type master_mem_limit: int
:type worker_count: int
:type worker_mem_limit: int
:type worker_cores: int
:type master_image: str
:type worker_image: str
:type notebook_image: str
:rtype: dict
"""
sp_master = spark_framework.spark_master_service(master_mem_limit, master_image)
sp_master['networks'].append('eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792')
sp_workers = spark_framework.spark_worker_service(worker_count, worker_mem_limit, worker_cores, worker_image)
for w in sp_workers:
w['networks'].append('eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792')
jupyter = jupyter_framework.spark_jupyter_notebook_service(notebook_mem_limit, worker_mem_limit, notebook_image)
jupyter['networks'].append('eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792')
jupyter['environment'].append(['NAMENODE_HOST', 'hdfs-namenode.hdfs'])
app = {
'name': name,
'version': 1,
'will_end': False,
'priority': 512,
'requires_binary': False,
'services': [
sp_master,
sp_workers[0],
jupyter,
] + sp_workers[1:]
}
return app
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zoe_lib.predefined_frameworks.hadoop as hadoop_framework
def hdfs_app(name='hdfs',
namenode_image='192.168.45.252:5000/zoerepo/hadoop-namenode',
datanode_count=3,
datanode_image='192.168.45.252:5000/zoerepo/hadoop-datanode'):
"""
:type name: str
:type namenode_image: str
:type datanode_count: int
:type datanode_image: str
:rtype: dict
"""
app = {
'name': name,
'version': 1,
'will_end': False,
'priority': 512,
'requires_binary': False,
'services': [
hadoop_framework.hadoop_namenode_service(namenode_image),
] + hadoop_framework.hadoop_datanode_service(datanode_count, datanode_image)
}
return app
def hdfs_client_app(name='hdfs-client',
image='192.168.45.252:5000/zoerepo/hadoop-client',
namenode='hdfs-namenode.hdfs',
user='root',
command='hdfs dfs -ls /'):
"""
:type name: str
:type image: str
:type user: str
:type namenode: str
:type command: str
:rtype: dict
"""
app = {
'name': name,
'version': 1,
'will_end': True,
'priority': 512,
'requires_binary': False,
'services': [
hadoop_framework.hadoop_client_service(image, namenode, user, command)
]
}
return app
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zoe_lib.predefined_frameworks.openmpi as openmpi_framework
def openmpi_app(name='openmpi-test', mpirun_commandline='', worker_count=4, worker_memory=(1024 ** 3)):
"""
:type name: str
:type mpirun_commandline: str
:type worker_count: int
:type worker_memory: int
:rtype: dict
"""
app = {
'name': name,
'version': 1,
'will_end': True,
'priority': 512,
'requires_binary': True,
'services': []
}
for i in range(worker_count):
proc = openmpi_framework.openmpi_worker_service(i, worker_memory)
app['services'].append(proc)
proc = openmpi_framework.openmpi_mpirun_service(mpirun_commandline, worker_memory)
app['services'].append(proc)
return app
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zoe_lib.predefined_frameworks.jupyter_spark as jupyter_spark_framework
import zoe_lib.predefined_frameworks.spark as spark_framework
def spark_jupyter_notebook_app(name='spark-jupyter',
notebook_mem_limit=4 * (1024 ** 3),
master_mem_limit=512 * (1024 ** 2),
worker_count=2,
worker_mem_limit=12 * (1024 ** 3),
worker_cores=6,
master_image='192.168.45.252:5000/zoerepo/spark-master',
worker_image='192.168.45.252:5000/zoerepo/spark-worker',
notebook_image='192.168.45.252:5000/zoerepo/spark-jupyter-notebook'):
"""
:type name: str
:type notebook_mem_limit: int
:type master_mem_limit: int
:type worker_count: int
:type worker_mem_limit: int
:type worker_cores: int
:type master_image: str
:type worker_image: str
:type notebook_image: str
:rtype: dict
"""
sp_master = spark_framework.spark_master_service(master_mem_limit, master_image)
sp_workers = spark_framework.spark_worker_service(worker_count, worker_mem_limit, worker_cores, worker_image)
app = {
'name': name,
'version': 1,
'will_end': False,
'priority': 512,
'requires_binary': False,
'services': [
sp_master,
jupyter_spark_framework.spark_jupyter_notebook_service(notebook_mem_limit, worker_mem_limit, notebook_image)
] + sp_workers
}
return app
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zoe_lib.predefined_frameworks.spark as spark_framework
def spark_submit_app(name='spark-submit',
master_mem_limit=4 * 1024 * 1024 * 1024,
worker_count=3,
worker_mem_limit=8 * 1024 * 1024 * 1024,
worker_cores=4,
master_image='192.168.45.252:5000/zoerepo/spark-master',
worker_image='192.168.45.252:5000/zoerepo/spark-worker',
submit_image='192.168.45.252:5000/zoerepo/spark-submit',
commandline='wordcount.py hdfs://192.168.45.157/datasets/gutenberg_big_2x.txt hdfs://192.168.45.157/tmp/cntwdc1'):
"""
:type name: str
:type master_mem_limit: int
:type worker_count: int
:type worker_mem_limit: int
:type worker_cores: int
:type master_image: str
:type worker_image: str
:type submit_image: str
:type commandline: str
:rtype: dict
"""
app = {
'name': name,
'version': 1,
'will_end': False,
'priority': 512,
'requires_binary': True,
'services': [
spark_framework.spark_master_service(master_mem_limit, master_image),
spark_framework.spark_submit_service(master_mem_limit, worker_mem_limit, submit_image, commandline)
] + spark_framework.spark_worker_service(worker_count, worker_mem_limit, worker_cores, worker_image)
}
return app
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_lib.predefined_frameworks.utils import sleeper_service
def sleeper_app(name='sleeper', sleep_duration=5):
"""
:param name:
:param sleep_duration:
:rtype: dict
"""
app = {
'name': name,
'version': 1,
'will_end': True,
'priority': 512,
'requires_binary': False,
'services': [
sleeper_service(sleep_duration)
]
}
return app
# Copyright (c) 2015, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.