Commit b4c70be0 authored by Daniele Venzano's avatar Daniele Venzano

Add options needed to run a development version in parallel to a production...

Add options needed to run a development version in parallel to a production version on the same network
parent 96d0c457
......@@ -12,5 +12,8 @@ conf = {
'check_health': 30,
'notebook_max_age_no_activity': 24,
'notebook_warning_age_no_activity': 2,
'email_task_interval': 300
'email_task_interval': 300,
'client_rpyc_autodiscovery': True,
'client_rpyc_server': None,
'client_rpyc_port': None
}
#!/usr/bin/env python3
import argparse
import asyncio
import logging
log = logging.getLogger('zoe')
import signal
from zoe_scheduler.rpyc_service import ZoeSchedulerRPCService
from zoe_scheduler.rpyc_server import RPyCAsyncIOServer
from zoe_scheduler.scheduler import zoe_sched
loop = None
rpyc_server = None
def sigint_handler():
log.warning('CTRL-C detected, terminating event loop...')
......@@ -19,12 +24,27 @@ def sigint_handler():
except RuntimeError:
pass
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
def process_arguments() -> argparse.Namespace:
argparser = argparse.ArgumentParser(description="Zoe Scheduler - Container Analytics as a Service scheduling component")
argparser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
argparser.add_argument('--rpyc-no-auto-register', action='store_true', help='Do not register automatically in the RPyC registry')
return argparser.parse_args()
def main():
global loop, rpyc_server
args = process_arguments()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('asyncio').setLevel(logging.INFO)
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger('asyncio').setLevel(logging.WARNING)
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('asyncio').setLevel(logging.INFO)
logging.getLogger('rpyc').setLevel(logging.WARNING)
log = logging.getLogger('zoe')
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGINT, sigint_handler)
......@@ -36,3 +56,6 @@ if __name__ == "__main__":
loop.run_forever()
loop.close()
if __name__ == "__main__":
main()
#!/usr/bin/env python3
import argparse
import logging
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop, PeriodicCallback
from zoe_web import app
from zoe_web.emails import email_task
from common.configuration import conf
DEBUG = True
log = logging.getLogger("zoe_web")
def process_arguments() -> argparse.Namespace:
argparser = argparse.ArgumentParser(description="Zoe Web - Container Analytics as a Service web client")
argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
argparser.add_argument('--rpyc-server', default=None, help='Specify an RPyC server instead of using autodiscovery')
argparser.add_argument('--rpyc-port', default=4000, type=int, help='Specify an RPyC server port, default is 4000')
return argparser.parse_args()
def main():
if DEBUG:
args = process_arguments()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("tornado").setLevel(logging.WARNING)
print("Starting app...")
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("tornado").setLevel(logging.WARNING)
if args.rpyc_server is None:
conf['client_rpyc_autodiscovery'] = True
else:
conf['client_rpyc_autodiscovery'] = False
conf['client_rpyc_server'] = args.rpyc_server
conf['client_rpyc_port'] = args.rpyc_port
log.info("Starting HTTP server...")
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
http_server = HTTPServer(WSGIContainer(app))
http_server.listen(4000, "0.0.0.0")
http_server.listen(5000, "0.0.0.0")
ioloop = IOLoop.instance()
PeriodicCallback(email_task, int(conf["email_task_interval"]) * 1000).start()
ioloop.start()
# PeriodicCallback(email_task, int(conf["email_task_interval"]) * 1000).start()
try:
ioloop.start()
except KeyboardInterrupt:
print("CTRL-C detected, terminating")
if __name__ == "__main__":
......
from zoe_client.client import ZoeClient
from zoe_client.client import ZoeClient, get_zoe_client
......@@ -10,6 +10,7 @@ from common.application_resources import SparkApplicationResources
from common.status import PlatformStatusReport, ApplicationStatusReport
from common.exceptions import UserIDDoesNotExist, ApplicationStillRunning
import common.object_storage as storage
from common.configuration import conf
REGISTRY = "10.0.0.2:5000"
MASTER_IMAGE = REGISTRY + "/zoe/spark-master-1.4.1:1.2"
......@@ -208,3 +209,10 @@ class ZoeClient:
return None
else:
return self.server.log_get(container_id)
def get_zoe_client():
if conf['client_rpyc_autodiscovery']:
return ZoeClient()
else:
return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])
from flask import Flask
from zoe_web.api import api
from zoe_web.web import web
from zoe_web.api import api_bp
from zoe_web.web import web_bp
app = Flask(__name__)
app.register_blueprint(web, url_prefix='/web')
app.register_blueprint(api, url_prefix='/web/api')
app.register_blueprint(web_bp, url_prefix='/web')
app.register_blueprint(api_bp, url_prefix='/web/api')
app.secret_key = b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
from flask import Blueprint, abort
api = Blueprint('api', __name__)
api_bp = Blueprint('api', __name__)
@api.route('/status/basic')
@api_bp.route('/status/basic')
def basic_status():
abort(404)
from flask import Blueprint, abort, render_template
from flask import Blueprint
from zoe_client import ZoeClient
import zoe_web.web.utils as web_utils
web = Blueprint('web', __name__, template_folder='templates', static_folder='static')
@web.route('/')
def index():
return render_template('index.html')
@web.route('/home')
def home():
client = ZoeClient()
user = web_utils.check_user(client)
template_vars = {
"user_id": user.id,
"email": user.email
}
return render_template('home.html', **template_vars)
web_bp = Blueprint('web', __name__, template_folder='templates', static_folder='static')
from flask import render_template
from zoe_client import get_zoe_client
from zoe_web.web import web_bp
import zoe_web.web.utils as web_utils
@web_bp.route('/')
def index():
return render_template('index.html')
@web_bp.route('/home')
def home():
client = get_zoe_client()
user = web_utils.check_user(client)
template_vars = {
"user_id": user.id,
"email": user.email
}
return render_template('home.html', **template_vars)
from flask import session, redirect, url_for
from zoe_client import ZoeClient
from zoe_web.web import web
from zoe_web.web import web_bp
from common.configuration import conf
def get_zoe_client():
if conf['client_rpyc_autodiscovery']:
return ZoeClient()
else:
return ZoeClient(conf['client_rpyc_server'], conf['client_rpyc_port'])
def check_user(zoeclient: ZoeClient):
if 'user_id' not in session:
return redirect(url_for(web.index))
return redirect(url_for(web_bp.index))
if not zoeclient.user_check(session['user_id']):
return redirect(url_for('index'))
......@@ -4,23 +4,15 @@ from argparse import ArgumentParser, Namespace
import logging
from zipfile import is_zipfile
from zoe_client import ZoeClient
from zoe_client import get_zoe_client
from common.state import create_tables
from common.configuration import conf
argparser = None
def get_zoe_client(args):
if args.rpyc_server is None:
return ZoeClient()
if args.rpyc_server is not None and args.rpyc_port is None:
return ZoeClient(args.rpyc_server)
else:
return ZoeClient(args.rpyc_server, args.rpyc_port)
def status_cmd(_):
client = ZoeClient()
client = get_zoe_client()
status_report = client.platform_status()
print(status_report)
......@@ -30,25 +22,25 @@ def setup_db_cmd(_):
def user_new_cmd(args):
client = ZoeClient()
client = get_zoe_client()
user = client.user_new(args.email)
print("New user ID: {}".format(user.id))
def user_get_cmd(args):
client = ZoeClient()
client = get_zoe_client()
user = client.user_get(args.email)
print("User ID: {}".format(user.email))
def spark_cluster_new_cmd(args):
client = ZoeClient()
client = get_zoe_client()
application_id = client.spark_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name)
print("Spark application added with ID: {}".format(application_id))
def spark_notebook_new_cmd(args):
client = ZoeClient()
client = get_zoe_client()
application_id = client.spark_notebook_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name)
print("Spark application added with ID: {}".format(application_id))
......@@ -57,13 +49,13 @@ def spark_submit_new_cmd(args):
if not is_zipfile(args.file):
print("Error: the file specified is not a zip archive")
return
client = ZoeClient()
client = get_zoe_client()
application_id = client.spark_submit_application_new(args.user_id, args.worker_count, args.executor_memory, args.executor_cores, args.name, args.file)
print("Spark application added with ID: {}".format(application_id))
def run_spark_cmd(args):
client = ZoeClient()
client = get_zoe_client()
application = client.application_get(args.id)
if application is None:
print("Error: application {} does not exist".format(args.id))
......@@ -77,7 +69,7 @@ def run_spark_cmd(args):
def app_rm_cmd(args):
client = ZoeClient()
client = get_zoe_client()
application = client.application_get(args.id)
if application is None:
print("Error: application {} does not exist".format(args.id))
......@@ -94,7 +86,7 @@ def app_rm_cmd(args):
def app_inspect_cmd(args):
client = ZoeClient()
client = get_zoe_client()
application = client.application_get(args.id)
if application is None:
print("Error: application {} does not exist".format(args.id))
......@@ -104,7 +96,7 @@ def app_inspect_cmd(args):
def app_list_cmd(args):
client = ZoeClient()
client = get_zoe_client()
applications = client.application_list(args.id)
if len(applications) > 0:
print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
......@@ -113,7 +105,7 @@ def app_list_cmd(args):
def exec_kill_cmd(args):
client = ZoeClient()
client = get_zoe_client()
execution = client.execution_get(args.id)
if execution is None:
print("Error: execution {} does not exist".format(args.id))
......@@ -122,7 +114,7 @@ def exec_kill_cmd(args):
def log_get_cmd(args):
client = ZoeClient()
client = get_zoe_client()
log = client.log_get(args.id)
if log is None:
print("Error: No log found for container ID {}".format(args.id))
......@@ -130,6 +122,7 @@ def log_get_cmd(args):
def process_arguments() -> Namespace:
global argparser
argparser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
argparser.add_argument('--rpyc-server', default=None, help='Specify an RPyC server instead of using autodiscovery')
......@@ -213,7 +206,17 @@ def main():
else:
logging.basicConfig(level=logging.INFO)
args.func(args)
if args.rpyc_server is None:
conf['client_rpyc_autodiscovery'] = True
else:
conf['client_rpyc_autodiscovery'] = False
conf['client_rpyc_server'] = args.rpyc_server
conf['client_rpyc_port'] = args.rpyc_port
args.func(args)
main()
if __name__ == "__main__":
try:
main()
except AttributeError:
argparser.print_help()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment