Add options needed to run a development version in parallel to a production...

Add options needed to run a development version in parallel to a production version on the same network

common/configuration.py

@@ -12,5 +12,8 @@ conf = {
     'check_health': 30,
     'notebook_max_age_no_activity': 24,
     'notebook_warning_age_no_activity': 2,
-    'email_task_interval': 300
+    'email_task_interval': 300,
+    'client_rpyc_autodiscovery': True,
+    'client_rpyc_server': None,
+    'client_rpyc_port': None
 }

zoe-scheduler.py

 #!/usr/bin/env python3
 
+import argparse
 import asyncio
 import logging
+log = logging.getLogger('zoe')
 import signal
 
 from zoe_scheduler.rpyc_service import ZoeSchedulerRPCService
 from zoe_scheduler.rpyc_server import RPyCAsyncIOServer
 from zoe_scheduler.scheduler import zoe_sched
 
+loop = None
+rpyc_server = None
+
 
 def sigint_handler():
     log.warning('CTRL-C detected, terminating event loop...')
@@ -19,12 +24,27 @@ def sigint_handler():
     except RuntimeError:
         pass
 
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
+
+def process_arguments() -> argparse.Namespace:
+    argparser = argparse.ArgumentParser(description="Zoe Scheduler - Container Analytics as a Service scheduling component")
+    argparser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
+    argparser.add_argument('--rpyc-no-auto-register', action='store_true', help='Do not register automatically in the RPyC registry')
+
+    return argparser.parse_args()
+
+
+def main():
+    global loop, rpyc_server
+    args = process_arguments()
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger('asyncio').setLevel(logging.INFO)
+    else:
+        logging.basicConfig(level=logging.INFO)
+        logging.getLogger('asyncio').setLevel(logging.WARNING)
+
     logging.getLogger('requests').setLevel(logging.WARNING)
-    logging.getLogger('asyncio').setLevel(logging.INFO)
     logging.getLogger('rpyc').setLevel(logging.WARNING)
-    log = logging.getLogger('zoe')
 
     loop = asyncio.get_event_loop()
     loop.add_signal_handler(signal.SIGINT, sigint_handler)
@@ -36,3 +56,6 @@ if __name__ == "__main__":
     loop.run_forever()
 
     loop.close()
+
+if __name__ == "__main__":
+    main()

zoe-web.py

 #!/usr/bin/env python3
 
+import argparse
 import logging
 from tornado.wsgi import WSGIContainer
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 from zoe_web import app
-from zoe_web.emails import email_task
 
 from common.configuration import conf
 
-DEBUG = True
 log = logging.getLogger("zoe_web")
 
 
+def process_arguments() -> argparse.Namespace:
+    argparser = argparse.ArgumentParser(description="Zoe Web - Container Analytics as a Service web client")
+    argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
+    argparser.add_argument('--rpyc-server', default=None, help='Specify an RPyC server instead of using autodiscovery')
+    argparser.add_argument('--rpyc-port', default=4000, type=int, help='Specify an RPyC server port, default is 4000')
+
+    return argparser.parse_args()
+
+
 def main():
-    if DEBUG:
+    args = process_arguments()
+    if args.debug:
         logging.basicConfig(level=logging.DEBUG)
-        logging.getLogger("requests").setLevel(logging.WARNING)
-        logging.getLogger("tornado").setLevel(logging.WARNING)
-
-    print("Starting app...")
+    else:
+        logging.basicConfig(level=logging.INFO)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("tornado").setLevel(logging.WARNING)
+
+    if args.rpyc_server is None:
+        conf['client_rpyc_autodiscovery'] = True
+    else:
+        conf['client_rpyc_autodiscovery'] = False
+        conf['client_rpyc_server'] = args.rpyc_server
+        conf['client_rpyc_port'] = args.rpyc_port
+
+    log.info("Starting HTTP server...")
     app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
 
     http_server = HTTPServer(WSGIContainer(app))
-    http_server.listen(4000, "0.0.0.0")
+    http_server.listen(5000, "0.0.0.0")
     ioloop = IOLoop.instance()
-    PeriodicCallback(email_task, int(conf["email_task_interval"]) * 1000).start()
-    ioloop.start()
+#    PeriodicCallback(email_task, int(conf["email_task_interval"]) * 1000).start()
+    try:
+        ioloop.start()
+    except KeyboardInterrupt:
+        print("CTRL-C detected, terminating")
 
 
 if __name__ == "__main__":

zoe_client/__init__.py

-from zoe_client.client import ZoeClient
+from zoe_client.client import ZoeClient, get_zoe_client

zoe_client/client.py

@@ -10,6 +10,7 @@ from common.application_resources import SparkApplicationResources
 from common.status import PlatformStatusReport, ApplicationStatusReport
 from common.exceptions import UserIDDoesNotExist, ApplicationStillRunning
 import common.object_storage as storage
+from common.configuration import conf
 
 REGISTRY = "10.0.0.2:5000"
 MASTER_IMAGE = REGISTRY + "/zoe/spark-master-1.4.1:1.2"
@@ -208,3 +209,10 @@ class ZoeClient:
             return None
         else:
             return self.server.log_get(container_id)
+
+
+def get_zoe_client():
+    if conf['client_rpyc_autodiscovery']:
+        return ZoeClient()
+    else:
+        return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])

zoe_web/__init__.py

 from flask import Flask
-from zoe_web.api import api
-from zoe_web.web import web
+from zoe_web.api import api_bp
+from zoe_web.web import web_bp
 
 app = Flask(__name__)
-app.register_blueprint(web, url_prefix='/web')
-app.register_blueprint(api, url_prefix='/web/api')
+app.register_blueprint(web_bp, url_prefix='/web')
+app.register_blueprint(api_bp, url_prefix='/web/api')
 
 app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"

zoe_web/api/__init__.py

 from flask import Blueprint, abort
 
-api = Blueprint('api', __name__)
+api_bp = Blueprint('api', __name__)
 
 
-@api.route('/status/basic')
+@api_bp.route('/status/basic')
 def basic_status():
     abort(404)

zoe_web/web/__init__.py

-from flask import Blueprint, abort, render_template
+from flask import Blueprint
 
-from zoe_client import ZoeClient
-import zoe_web.web.utils as web_utils
-
-web = Blueprint('web', __name__, template_folder='templates', static_folder='static')
-
-
-@web.route('/')
-def index():
-    return render_template('index.html')
-
-
-@web.route('/home')
-def home():
-    client = ZoeClient()
-    user = web_utils.check_user(client)
-    template_vars = {
-        "user_id": user.id,
-        "email": user.email
-    }
-    return render_template('home.html', **template_vars)
+web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')

zoe_web/web/start.py

+from flask import render_template
+
+from zoe_client import get_zoe_client
+from zoe_web.web import web_bp
+import zoe_web.web.utils as web_utils
+
+
+@web_bp.route('/')
+def index():
+    return render_template('index.html')
+
+
+@web_bp.route('/home')
+def home():
+    client = get_zoe_client()
+    user = web_utils.check_user(client)
+    template_vars = {
+        "user_id": user.id,
+        "email": user.email
+    }
+    return render_template('home.html', **template_vars)

zoe_web/web/utils.py

 from flask import session, redirect, url_for
 
 from zoe_client import ZoeClient
-from zoe_web.web import web
+from zoe_web.web import web_bp
+from common.configuration import conf
+
+
+def get_zoe_client():
+    if conf['client_rpyc_autodiscovery']:
+        return ZoeClient()
+    else:
+        return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])
 
 
 def check_user(zoeclient: ZoeClient):
     if 'user_id' not in session:
-        return redirect(url_for(web.index))
+        return redirect(url_for(web_bp.index))
     if not zoeclient.user_check(session['user_id']):
         return redirect(url_for('index'))

zoectl.py

@@ -4,23 +4,15 @@ from argparse import ArgumentParser, Namespace
 import logging
 from zipfile import is_zipfile
 
-from zoe_client import ZoeClient
+from zoe_client import get_zoe_client
 from common.state import create_tables
+from common.configuration import conf
 
 argparser = None
 
 
-def get_zoe_client(args):
-    if args.rpyc_server is None:
-        return ZoeClient()
-    if args.rpyc_server is not None and args.rpyc_port is None:
-        return ZoeClient(args.rpyc_server)
-    else:
-        return ZoeClient(args.rpyc_server, args.rpyc_port)
-
-
 def status_cmd(_):
-    client = ZoeClient()
+    client = get_zoe_client()
     status_report = client.platform_status()
     print(status_report)
 
@@ -30,25 +22,25 @@ def setup_db_cmd(_):
 
 
 def user_new_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     user = client.user_new(args.email)
     print("New user ID: {}".format(user.id))
 
 
 def user_get_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     user = client.user_get(args.email)
     print("User ID: {}".format(user.email))
 
 
 def spark_cluster_new_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     application_id = client.spark_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name)
     print("Spark application added with ID: {}".format(application_id))
 
 
 def spark_notebook_new_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     application_id = client.spark_notebook_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name)
     print("Spark application added with ID: {}".format(application_id))
 
@@ -57,13 +49,13 @@ def spark_submit_new_cmd(args):
     if not is_zipfile(args.file):
         print("Error: the file specified is not a zip archive")
         return
-    client = ZoeClient()
+    client = get_zoe_client()
     application_id = client.spark_submit_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name, args.file)
     print("Spark application added with ID: {}".format(application_id))
 
 
 def run_spark_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     application = client.application_get(args.id)
     if application is None:
         print("Error: application {} does not exist".format(args.id))
@@ -77,7 +69,7 @@ def run_spark_cmd(args):
 
 
 def app_rm_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     application = client.application_get(args.id)
     if application is None:
         print("Error: application {} does not exist".format(args.id))
@@ -94,7 +86,7 @@ def app_rm_cmd(args):
 
 
 def app_inspect_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     application = client.application_get(args.id)
     if application is None:
         print("Error: application {} does not exist".format(args.id))
@@ -104,7 +96,7 @@ def app_inspect_cmd(args):
 
 
 def app_list_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     applications = client.application_list(args.id)
     if len(applications) > 0:
         print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
@@ -113,7 +105,7 @@ def app_list_cmd(args):
 
 
 def exec_kill_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     execution = client.execution_get(args.id)
     if execution is None:
         print("Error: execution {} does not exist".format(args.id))
@@ -122,7 +114,7 @@ def exec_kill_cmd(args):
 
 
 def log_get_cmd(args):
-    client = ZoeClient()
+    client = get_zoe_client()
     log = client.log_get(args.id)
     if log is None:
         print("Error: No log found for container ID {}".format(args.id))
@@ -130,6 +122,7 @@ def log_get_cmd(args):
 
 
 def process_arguments() -> Namespace:
+    global argparser
     argparser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
     argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
     argparser.add_argument('--rpyc-server', default=None, help='Specify an RPyC server instead of using autodiscovery')
@@ -213,7 +206,17 @@ def main():
     else:
         logging.basicConfig(level=logging.INFO)
 
-    args.func(args)
+    if args.rpyc_server is None:
+        conf['client_rpyc_autodiscovery'] = True
+    else:
+        conf['client_rpyc_autodiscovery'] = False
+        conf['client_rpyc_server'] = args.rpyc_server
+        conf['client_rpyc_port'] = args.rpyc_port
 
+    args.func(args)
 
-main()
+if __name__ == "__main__":
+    try:
+        main()
+    except AttributeError:
+        argparser.print_help()