Move everything back to one repository for simpler deployment and easier version management

.gitignore

@@ -59,3 +59,4 @@ docs/_build/
 target/
 .idea/
 state.zoe
+*.conf

.travis.yml

+language: python
+python:
+  - "3.4"
+  - "3.5"
+install:
+  - pip install -r requirements.txt
+before_script:
+script:
+  - PYTHONPATH=. py.test --cov=zoe_cmd --cov=zoe_lib --cov=zoe_web --cov=zoe_scheduler --cov=zoe_observer

README.rst

@@ -3,28 +3,36 @@ Zoe - Container-based Analytics as a Service
 
 Zoe provides a simple way to provision data analytics applications using Docker Swarm.
 
-This is the main repository, it contains the documentation and a number of scripts, useful to install and develop Zoe.
-
-We are in the process of doing a major refactoring of the entire codebase and the HEAD version is not fully tested.
-
 Resources:
 
 -  Main website: http://zoe-analytics.eu
 -  Documentation: http://docs.zoe-analytics.eu
--  How to install: http://zoe-analytics.readthedocs.org/en/latest/install.html (refers to the 0.8.92 version)
-
-Zoe is a distributed application and each component is developed in a separate Git repository.
-
--  Zoe clients: https://github.com/DistributedSystemsGroup/zoe-client
--  Zoe scheduler: https://github.com/DistributedSystemsGroup/zoe-scheduler
+-  How to install: http://zoe-analytics.readthedocs.org/en/latest/install.html
 
 Zoe can use any Docker image, but we provide some for the pre-configured applications available in the client (Spark and HDFS):
 
 -  Docker images: https://github.com/DistributedSystemsGroup/zoe-docker-images
 
-|Documentation Status|
+Repository contents
+-------------------
+
+- `contrib`: supervisord config files
+- `docs`: Sphinx documentation
+- `scripts`: Scripts used to test Zoe images outside of Zoe
+- `zoe_cmd`: Command-line client
+- `zoe_lib`: Client-side library, contains also some modules needed by the observer and the scheduler processes
+- `zoe_observer`: The Observer process that monitors Swarm and informs the scheduler of various events
+- `zoe_scheduler`: The core of Zoe, the server process that listens for client requests and creates the containers on Swarm
+- `zoe_web`: The web client interface
+
+|Travis build| |Documentation Status| |Requirements Status|
 
 Zoe is licensed under the terms of the Apache 2.0 license.
 
 .. |Documentation Status| image:: https://readthedocs.org/projects/zoe-analytics/badge/?version=latest
    :target: https://readthedocs.org/projects/zoe-analytics/?badge=latest
+.. |Requirements Status| image:: https://requires.io/github/DistributedSystemsGroup/zoe/requirements.svg?branch=master
+   :target: https://requires.io/github/DistributedSystemsGroup/zoe/requirements/?branch=master
+   :alt: Requirements Status
+.. |Travis build| image:: https://travis-ci.org/DistributedSystemsGroup/zoe.svg
+   :target: https://travis-ci.org/DistributedSystemsGroup/zoe
\ No newline at end of file

basic_usage.sh

+#!/usr/bin/env bash
+
+set -e
+
+echo "Get statistics"
+./zoe.py stats
+echo "Get superuser"
+./zoe.py user-get zoeadmin
+echo "Create a new user"
+./zoe.py user-new --name test --password test --role guest
+echo "Delete a user"
+./zoe.py user-rm test
+echo "Export an application template"
+./zoe.py pre-app-export hdfs > /tmp/zoe-hdfs.json
+echo "Upload the template as a new application"
+./zoe.py app-new /tmp/zoe-hdfs.json
+echo "Get the application back from Zoe"
+./zoe.py app-get hdfs
+echo "Delete the app"
+./zoe.py app-rm hdfs

requirements.txt

+Flask>=0.10.1
+python-dateutil>=2.4.2
+pytest
+pytest-cov
+requests
+SQLAlchemy>=1.0.8
+pyzmq>=14.0.1
+docker-py>=1.6.0
+tornado>=4.3
+kazoo
+passlib

zoe-aio.py

+#!/usr/bin/python3
+
+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import multiprocessing
+import time
+
+from zoe_lib.exceptions import ZoeException
+
+log = logging.getLogger('ZoeAIO')
+
+
+def start_sched():
+    from zoe_scheduler.entrypoint import main
+    main()
+
+
+def start_observer():
+    from zoe_observer.entrypoint import main
+    main()
+
+sched = multiprocessing.Process(target=start_sched, name='scheduler')
+obs = multiprocessing.Process(target=start_observer, name='observer')
+
+sched.start()
+obs.start()
+
+try:
+    while True:
+        procs = multiprocessing.active_children()
+        if len(procs) == 0:
+            raise ZoeException('All Zoe processes crashed, exiting')
+        elif len(procs) == 1:
+            raise ZoeException('A process crashed, terminating {} that is still alive'.format(procs[0].name))
+        else:
+            time.sleep(1)
+except ZoeException as e:
+    log.error(str(e))
+except KeyboardInterrupt:
+    log.info('CTRL-C detected, exiting...')
+finally:
+    sched.terminate()
+    obs.terminate()

zoe-observer.py

+#!/usr/bin/python3
+
+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zoe_observer.entrypoint import main
+
+if __name__ == '__main__':
+    main()

zoe-scheduler.py

+#!/usr/bin/python3
+
+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zoe_scheduler.entrypoint import main
+
+if __name__ == '__main__':
+    main()

zoe-web.py

+#!/usr/bin/env python3
+
+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zoe_web.entrypoint import zoe_web_main
+
+if __name__ == "__main__":
+    zoe_web_main()

zoe.py

+#!/usr/bin/env python3
+
+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zoe_cmd.entrypoint import zoe
+
+if __name__ == "__main__":
+    zoe()

zoe_cmd/entrypoint.py

+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the entrypoint for the commandline Zoe client
+"""
+
+import json
+import logging
+import os
+import sys
+from argparse import ArgumentParser, Namespace, FileType, RawDescriptionHelpFormatter
+from pprint import pprint
+
+from zoe_cmd import utils
+from zoe_lib.applications import ZoeApplicationAPI
+from zoe_lib.containers import ZoeContainerAPI
+from zoe_lib.exceptions import ZoeAPIException
+from zoe_lib.executions import ZoeExecutionsAPI
+from zoe_lib.predefined_apps import hadoop, spark, lab_spark, test_sleep
+from zoe_lib.query import ZoeQueryAPI
+from zoe_lib.users import ZoeUserAPI
+
+PREDEFINED_APPS = {}
+for mod in [hadoop, spark, lab_spark, test_sleep]:
+    for app_name, val in mod.__dict__.items():
+        if callable(val) and "_app" in app_name:
+            PREDEFINED_APPS[app_name[:-4]] = val
+
+
+def stats_cmd(_):
+    api = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    stats = api.query('stats swarm')
+    pprint(stats)
+
+
+def user_new_cmd(args):
+    api = ZoeUserAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    name = args.name
+    password = args.password
+    role = args.role
+    if role not in ['admin', 'user', 'guest']:
+        print("Role must be one of admin, user, guest)")
+        return
+    user_id = api.create(name, password, role)
+    print("New user ID: {}".format(user_id))
+
+
+def user_get_cmd(args):
+    api = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = api.query('user', name=args.name)
+    if len(data) == 0:
+        print('No such user')
+    else:
+        user = data[0]
+        print('User: {}'.format(user['name']))
+        print('User ID: {}'.format(user['id']))
+        print('Role: {}'.format(user['role']))
+        print('Gateway URLs: {}'.format(user['gateway_urls']))
+
+
+def user_delete_cmd(args):
+    api_user = ZoeUserAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    api_query = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = api_query.query('user', name=args.name)
+    if len(data) == 0:
+        print('No such user')
+    else:
+        for user in data:
+            print('Deleting user {}'.format(user['name']))
+            api_user.delete(user['id'])
+
+
+def user_list_cmd(_):
+    api_query = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    users = api_query.query('user')
+    for user in users:
+        print('<- User {} ->'.format(user['id']))
+        print('User: {}'.format(user['name']))
+        print('Role: {}'.format(user['role']))
+        print('Gateway URLs: {}'.format(user['gateway_urls']))
+
+
+def pre_app_list_cmd(_):
+    for a in PREDEFINED_APPS.keys():
+        print(a)
+
+
+def pre_app_export_cmd(args):
+    try:
+        func = PREDEFINED_APPS[args.app_name]
+    except KeyError:
+        print('Application not found')
+    else:
+        app = func()
+        json.dump(app, sys.stdout, sort_keys=True, indent=4)
+        print()
+
+
+def app_new_cmd(args):
+    app_descr = json.load(args.jsonfile)
+    api = ZoeApplicationAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    try:
+        app_id = api.create(app_descr)
+    except ZoeAPIException as e:
+        print("Invalid application description: %s" % e.message)
+        return
+    print("Application added with ID: {}".format(app_id))
+
+
+def app_get_cmd(args):
+    api_query = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = api_query.query('application', name=args.name)
+    if len(data) == 0:
+        print("no such application")
+    else:
+        for app in data:
+            pprint(app)
+
+
+def app_rm_cmd(args):
+    api_app = ZoeApplicationAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    api_query = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = api_query.query('application', name=args.name)
+    if len(data) == 0:
+        print('No such application')
+    else:
+        for app in data:
+            print('Deleting app {}'.format(app['name']))
+            api_app.delete(app['id'])
+
+
+def app_list_cmd(_):
+    api_query = ZoeQueryAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = api_query.query('application')
+    for app in data:
+        print('{} (ID: {})'.format(app['name'], app['id']))
+
+
+def exec_list_cmd(_):
+    exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    data = exec_api.list()
+    for e in data:
+        print('Execution {} (ID: {}): {}'.format(e['name'], e['id'], e['status']))
+
+
+def exec_start_cmd(args):
+    exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    ret = exec_api.execution_start(args.name, args.app_name)
+    print("Application scheduled successfully with ID {}, use the exec-get command to check its status".format(ret))
+
+
+def exec_get_cmd(args):
+    app_api = ZoeApplicationAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    cont_api = ZoeContainerAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    execution = exec_api.execution_get(args.id)
+    if execution is None:
+        print('Execution not found')
+    else:
+        print('Execution {} (ID: {})'.format(execution['name'], execution['id']))
+        print('Status: {}'.format(execution['status']))
+        print('Time started: {}'.format(execution['time_started']))
+        print('Time scheduled: {}'.format(execution['time_scheduled']))
+        print('Time finished: {}'.format(execution['time_finished']))
+        app = app_api.get(execution['application_id'])
+        print('Application name: {}'.format(app['name']))
+        for c_id in execution['containers']:
+            c = cont_api.get(c_id)
+            ip = list(c['ip_address'].values())[0]  # FIXME how to decide which network is the right one?
+            print('Container {} (ID: {})'.format(c['name'], c['id']))
+            for p in c['ports']:
+                print(' - {}: {}://{}:{}{}'.format(p['name'], p['protocol'], ip, p['port_number'], p['path']))
+
+
+def exec_kill_cmd(args):
+    exec_api = ZoeExecutionsAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    exec_api.terminate(args.id)
+
+
+def log_get_cmd(args):
+    cont_api = ZoeContainerAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    log = cont_api.log(args.container_id)
+    if log is None:
+        print("Error: No log found for container ID {}".format(args.container_id))
+        return
+    try:
+        print(log)
+    except BrokenPipeError:
+        pass
+
+
+def container_stats_cmd(args):
+    cont_api = ZoeContainerAPI(utils.zoe_url(), utils.zoe_user(), utils.zoe_pass())
+    stats = cont_api.stats(args.container_id)
+    print(stats)
+
+
+ENV_HELP_TEXT = '''To use this tool you need also to define three environment variables:
+ZOE_URL: point to the URL of the Zoe Scheduler (ex.: http://localhost:5000/
+ZOE_USER: the username used for authentication
+ZOE_PASS: the password used for authentication'''
+
+
+def process_arguments() -> Namespace:
+    parser = ArgumentParser(description="Zoe command-line client", epilog=ENV_HELP_TEXT, formatter_class=RawDescriptionHelpFormatter)
+    parser.add_argument('--debug', action='store_true', help='Enable debug output')
+
+    subparser = parser.add_subparsers()
+
+    argparser_stats = subparser.add_parser('stats', help="Show the platform statistics")
+    argparser_stats.set_defaults(func=stats_cmd)
+
+    argparser_user_new = subparser.add_parser('user-new', help="Create a new user")
+    argparser_user_new.add_argument('--name', help="User name")
+    argparser_user_new.add_argument('--password', help="Password")
+    argparser_user_new.add_argument('--role', help="Role (one of admin, user, guest)")
+    argparser_user_new.set_defaults(func=user_new_cmd)
+
+    argparser_user_get = subparser.add_parser('user-get', help="Retrieve information about a user")
+    argparser_user_get.add_argument('name', help="User name")
+    argparser_user_get.set_defaults(func=user_get_cmd)
+
+    argparser_user_delete = subparser.add_parser('user-rm', help="Removes a user from the system")
+    argparser_user_delete.add_argument('name', help="User name")
+    argparser_user_delete.set_defaults(func=user_delete_cmd)
+
+    argparser_user_list = subparser.add_parser('user-list', help='Lists all users defined in the system')
+    argparser_user_list.set_defaults(func=user_list_cmd)
+
+    argparser_pre_app_list = subparser.add_parser('pre-app-list', help='Lists the predefined application descriptions')
+    argparser_pre_app_list.set_defaults(func=pre_app_list_cmd)
+
+    argparser_pre_app_export = subparser.add_parser('pre-app-export', help='Export one of the predefined application descriptions in JSON (stdout)')
+    argparser_pre_app_export.add_argument('app_name', help='Predefined application name (use pre-app-list to see what is available')
+    argparser_pre_app_export.set_defaults(func=pre_app_export_cmd)
+
+    argparser_app_new = subparser.add_parser('app-new', help="Upload a JSON application description")
+    argparser_app_new.add_argument('jsonfile', type=FileType("r"), help='Application description')
+    argparser_app_new.set_defaults(func=app_new_cmd)
+
+    argparser_app_get = subparser.add_parser('app-get', help="Retrieve an already defined application description")
+    argparser_app_get.add_argument('name', help='The name of the application')
+    argparser_app_get.set_defaults(func=app_get_cmd)
+
+    argparser_app_rm = subparser.add_parser('app-rm', help="Delete an application")
+    argparser_app_rm.add_argument('name', help="Application name (will fail if there are running executions)")
+    argparser_app_rm.set_defaults(func=app_rm_cmd)
+
+    argparser_app_list = subparser.add_parser('app-ls', help="List all applications defined by the calling user")
+    argparser_app_list.set_defaults(func=app_list_cmd)
+
+    argparser_exec_start = subparser.add_parser('start', help="Start a previously registered application")
+    argparser_exec_start.add_argument('app_name', help="Name of the application to start")
+    argparser_exec_start.add_argument('name', help="Name of the execution")
+    argparser_exec_start.set_defaults(func=exec_start_cmd)
+
+    argparser_execution_get = subparser.add_parser('exec-get', help="Get execution status")
+    argparser_execution_get.add_argument('id', type=int, help="Execution id")
+    argparser_execution_get.set_defaults(func=exec_get_cmd)
+
+    argparser_app_list = subparser.add_parser('exec-ls', help="List all executions for the calling user")
+    argparser_app_list.set_defaults(func=exec_list_cmd)
+
+    argparser_execution_kill = subparser.add_parser('terminate', help="Terminates an execution")
+    argparser_execution_kill.add_argument('id', type=int, help="Execution id")
+    argparser_execution_kill.set_defaults(func=exec_kill_cmd)
+
+    argparser_log_get = subparser.add_parser('log-get', help="Retrieves the logs of a running container")
+    argparser_log_get.add_argument('container_id', type=int, help="Container id")
+    argparser_log_get.set_defaults(func=log_get_cmd)
+
+    argparser_container_stats = subparser.add_parser('container-stats', help="Retrieve statistics on a running container")
+    argparser_container_stats.add_argument('container_id', type=int, help="ID of the container")
+    argparser_container_stats.set_defaults(func=container_stats_cmd)
+
+    return parser, parser.parse_args()
+
+
+def zoe():
+    parser, args = process_arguments()
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return
+
+    if 'ZOE_URL' not in os.environ or 'ZOE_USER' not in os.environ or 'ZOE_PASS' not in os.environ:
+        parser.print_help()
+        return
+
+    try:
+        args.func(args)
+    except ZoeAPIException as e:
+        print(e.message)
+    sys.exit(0)

zoe_cmd/utils.py

+import os
+
+
+def zoe_url():
+    return os.environ['ZOE_URL']
+
+
+def zoe_user():
+    return os.environ['ZOE_USER']
+
+
+def zoe_pass():
+    return os.environ['ZOE_PASS']

zoe_lib/__init__.py

+"""
+This package contains all the Zoe client API.
+"""
+import requests
+import requests.exceptions
+
+from zoe_lib.version import ZOE_API_VERSION
+from zoe_lib.exceptions import ZoeAPIException
+
+
+class ZoeAPIBase:
+    def __init__(self, url, user, password):
+        self.url = url
+        self.user = user
+        self.password = password
+
+    def _rest_get(self, path: str):
+        url = self.url + '/api/' + ZOE_API_VERSION + path
+        try:
+            r = requests.get(url, auth=(self.user, self.password))
+        except requests.exceptions.Timeout:
+            raise ZoeAPIException('HTTP connection timeout')
+        except requests.exceptions.HTTPError:
+            raise ZoeAPIException('Invalid HTTP response')
+        except requests.exceptions.ConnectionError as e:
+            raise ZoeAPIException('Connection error: {}'.format(e))
+
+        try:
+            data = r.json()
+        except ValueError:
+            data = None
+        return data, r.status_code
+
+    def _rest_post(self, path, payload):
+        url = self.url + '/api/' + ZOE_API_VERSION + path
+        try:
+            r = requests.post(url, auth=(self.user, self.password), json=payload)
+        except requests.exceptions.Timeout:
+            raise ZoeAPIException('HTTP connection timeout')
+        except requests.exceptions.HTTPError:
+            raise ZoeAPIException('Invalid HTTP response')
+        except requests.