Refactor configuration

Split configuration into different modules for the scheduler and the clients

.gitignore

@@ -59,4 +59,5 @@ docs/_build/
 # PyBuilder
 target/
 .idea/
-zoe.conf
+zoe-scheduler.conf
+zoe-client.conf

common/configuration.py

-from configparser import ConfigParser
-
-ipcconf = {
-    'server': None,
-    'port': None,
-}
-
-config_paths = [
-    'zoe.conf',
-    '/etc/zoe/zoe.conf'
-]
-
-defaults = {
-    'docker': {
-        'swarm_manager_url': 'tcp://swarm.example.com:2380',
-        'private_registry': '10.1.0.1:5000'
-    },
-    'intervals': {
-        'status_refresh': 10,
-        'scheduler_task': 10,
-        'check_health': 30,
-        'notebook_max_age_no_activity': 24,
-        'notebook_warning_age_no_activity': 2
-    },
-    'db': {
-        'url': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe'
-    },
-    'smtp': {
-        'server': 'smtp.exmaple.com',
-        'user': 'zoe@exmaple.com',
-        'password': 'changeme'
-    },
-    'filesystem': {
-        'history_path': "/var/lib/zoe/history"
-    },
-    'flask': {
-        'secret_key': b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
-    },
-    'network': {
-        'scheduler_internal_server_port': 4390,
-        'scheduler_internal_hostname': '127.0.0.1'
-    }
-}
-
-_zoeconf = None
-
-
-class ZoeConfig(ConfigParser):
-    def __init__(self):
-        super().__init__(interpolation=None)
-        self.read_dict(defaults)
-
-    def write_defaults(self, fp):
-        tmp = ZoeConfig()
-        tmp.write(fp)
-
-    @property
-    def history_path(self) -> str:
-        return self.get('filesystem', 'history_path')
-
-    @property
-    def web_server_name(self) -> str:
-        return self.get('apache', 'web_server_name')
-
-    @property
-    def smtp_server(self) -> str:
-        return self.get('smtp', 'server')
-
-    @property
-    def smtp_user(self) -> str:
-        return self.get('smtp', 'user')
-
-    @property
-    def smtp_password(self) -> str:
-        return self.get('smtp', 'password')
-
-    @property
-    def notebook_warning_age_no_activity(self) -> int:
-        return self.getint('intervals', 'notebook_warning_age_no_activity')
-
-    @property
-    def notebook_max_age_no_activity(self) -> int:
-        return self.getint('intervals', 'notebook_max_age_no_activity')
-
-    @property
-    def interval_check_health(self) -> int:
-        return self.getint('intervals', 'check_health')
-
-    @property
-    def db_url(self) -> str:
-        return self.get('db', 'url')
-
-    @property
-    def interval_scheduler_task(self) -> int:
-        return self.getint('intervals', 'scheduler_task')
-
-    @property
-    def interval_status_refresh(self) -> int:
-        return self.getint('intervals', 'status_refresh')
-
-    @property
-    def docker_swarm_manager(self) -> str:
-        return self.get('docker', 'swarm_manager_url')
-
-    @property
-    def cookies_secret_key(self):
-        return self.get('flask', 'secret_key')
-
-    @property
-    def docker_private_registry(self) -> str:
-        return self.get('docker', 'private_registry')
-
-    @property
-    def scheduler_internal_server_port(self) -> int:
-        return self.getint('network', 'scheduler_internal_server_port')
-
-    @property
-    def scheduler_internal_hostname(self) -> str:
-        return self.get('network', 'scheduler_internal_hostname')
-
-
-def init(config_file=None) -> ZoeConfig:
-    global _zoeconf
-    _zoeconf = ZoeConfig()
-    if config_file is None:
-        _zoeconf.read(config_paths)
-    else:
-        _zoeconf.read_file(open(config_file))
-    return _zoeconf
-
-
-def zoeconf() -> ZoeConfig:
-    return _zoeconf

tests/conftest.py

 import json
+
 import pytest
 
 from zoe_scheduler.state import init as state_init, Base, AlchemySession
 from zoe_scheduler.state.application import ApplicationState
 from zoe_scheduler.application_description import ZoeApplication
-
-from common.configuration import init as conf_init, zoeconf
+from zoe_scheduler.configuration import init as conf_init, scheduler_conf
 
 
 def pytest_addoption(parser):
@@ -27,7 +27,7 @@ def configuration(environment):
 
 @pytest.fixture(scope='session')
 def state_connection(request, configuration):
-    engine = state_init(zoeconf().db_url)
+    engine = state_init(scheduler_conf().db_url)
     connection = engine.connect()
     trans = connection.begin()
 

tests/resources/zoe-local.conf

-[docker]
+[zoe_scheduler]
 swarm_manager_url = tcp://example.com:2380
-private_registry = 10.1.0.1:5000
+docker_private_registry = 10.1.0.1:5000
+status_refresh_interval = 10
+check_terminated_interval = 30
+db_connect = mysql+mysqlconnector://root@localhost/zoe
+storage_path = /tmp/zoe
+http_listen_address = 192.168.45.25
+http_listen_port = 4390
+ipc_listen_address = 127.0.0.1
+ipc_listen_port = 8723
 
-[intervals]
-check_health = 30
-notebook_max_age_no_activity = 24
-scheduler_task = 10
-notebook_warning_age_no_activity = 2
-status_refresh = 10
+[zoe_client]
+db_connect = mysql+mysqlconnector://root@localhost/zoe
+scheduler_ipc_address = localhost
+scheduler_ipc_port = 8723
 
-[filesystem]
-history_path = /tmp/history
-
-[flask]
-secret_key = b"\xc4\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
-
-[smtp]
-password = none
-user = none
-server = none
-
-[db]
-url = mysql+mysqlconnector://root@localhost/zoe
-
-[network]
-scheduler_internal_server_port = 4390
-scheduler_internal_hostname = 192.168.45.25
+[zoe_web]
+smtp_password = pass
+smtp_user = user
+smtp_server = smtp.example.com
+cookie_secret = \xc3\xb0\xa7\xaa\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
+web_server_name = localhost

tests/resources/zoe-travis.conf

-[docker]
+[zoe_scheduler]
 swarm_manager_url = tcp://example.com:2380
+docker_private_registry = 10.1.0.1:5000
+status_refresh_interval = 10
+check_terminated_interval = 30
+db_connect = mysql+mysqlconnector://root@localhost/zoe
+storage_path = /tmp/zoe
+http_listen_address = 192.168.45.25
+http_listen_port = 4390
+ipc_listen_address = 127.0.0.1
+ipc_listen_port = 8723
 
-[intervals]
-check_health = 30
-notebook_max_age_no_activity = 24
-scheduler_task = 10
-notebook_warning_age_no_activity = 2
-status_refresh = 10
+[zoe_client]
+db_connect = mysql+mysqlconnector://root@localhost/zoe
+scheduler_ipc_address = localhost
+scheduler_ipc_port = 8723
 
-[filesystem]
-history_path = /tmp/history
-
-[flask]
-secret_key = b"\xc4\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
-
-[smtp]
-password = none
-user = none
-server = none
-
-[db]
-url = mysql+mysqlconnector://root@localhost/zoe
+[zoe_web]
+smtp_password = pass
+smtp_user = user
+smtp_server = smtp.example.com
+cookie_secret = \xc3\xb0\xa7\xaa\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
+web_server_name = localhost

zoe-client.conf

+[zoe_client]
+db_connect = mysql+mysqlconnector://root@localhost/zoe
+scheduler_ipc_address = localhost
+scheduler_ipc_port = 8723
+
+[zoe_web]
+smtp_password = Daicu2Ze
+smtp_user = bigfoot.data@gmail.com
+smtp_server = smtp.gmail.com
+cookie_secret = \xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
+web_server_name = 192.168.45.25

zoe_client/client.py

@@ -5,7 +5,6 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from zoe_client.state import AlchemySession
 from zoe_client.ipc import ZoeIPCClient
-from common.configuration import zoeconf
 from zoe_client.entities import Execution, Application, User
 from zoe_client.state.user import UserState
 
@@ -15,7 +14,6 @@ log = logging.getLogger(__name__)
 class ZoeClient:
     def __init__(self, ipc_server='localhost', ipc_port=8723):
         self.ipc_server = ZoeIPCClient(ipc_server, ipc_port)
-        self.image_registry = zoeconf().docker_private_registry
         self.state = AlchemySession()
 
     # Applications

zoe_client/configuration.py

+from configparser import ConfigParser
+
+config_paths = [
+    'zoe.conf',
+    'zoe-client.conf',
+    '/etc/zoe/zoe.conf',
+    '/etc/zoe/zoe-client.conf'
+]
+
+defaults = {
+    'zoe_client': {
+        'db_connect': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe',
+        'scheduler_ipc_address': 'localhost',
+        'scheduler_ipc_port': 8723
+    },
+    'zoe_web': {
+        'smtp_server': 'smtp.exmaple.com',
+        'smtp_user': 'zoe@exmaple.com',
+        'smtp_password': 'changeme',
+        'cookie_secret': b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!",
+        'web_server_name': 'localhost'
+    }
+}
+
+_zoeconf = None
+
+
+class ZoeClientConfig(ConfigParser):
+    def __init__(self):
+        super().__init__(interpolation=None)
+        self.read_dict(defaults)
+
+    @staticmethod
+    def write_defaults(cls, fp):
+        tmp = cls()
+        tmp.write(fp)
+
+    @property
+    def db_url(self) -> str:
+        return self.get('zoe_client', 'db_connect')
+
+    @property
+    def ipc_server(self) -> str:
+        return self.get('zoe_client', 'scheduler_ipc_address')
+
+    @property
+    def ipc_port(self) -> int:
+        return self.getint('zoe_client', 'scheduler_ipc_port')
+
+    @property
+    def web_server_name(self) -> str:
+        return self.get('zoe_web', 'web_server_name')
+
+    @property
+    def smtp_server(self) -> str:
+        return self.get('zoe_web', 'smtp_server')
+
+    @property
+    def smtp_user(self) -> str:
+        return self.get('zoe_web', 'smtp_user')
+
+    @property
+    def smtp_password(self) -> str:
+        return self.get('zoe_web', 'smtp_password')
+
+    @property
+    def cookies_secret_key(self):
+        return self.get('zoe_web', 'cookie_secret')
+
+
+def conf_init(config_file=None) -> ZoeClientConfig:
+    global _zoeconf
+    _zoeconf = ZoeClientConfig()
+    if config_file is None:
+        _zoeconf.read(config_paths)
+    else:
+        _zoeconf.read_file(open(config_file))
+    return _zoeconf
+
+
+def client_conf() -> ZoeClientConfig:
+    return _zoeconf

zoe_client/entrypoint.py

@@ -3,46 +3,44 @@ import json
 import logging
 from zipfile import is_zipfile
 from pprint import pprint
+import sys
 
 from zoe_client import ZoeClient
-from common.configuration import init as conf_init, zoeconf
-from zoe_client.state import create_tables, init as state_init
+from zoe_client.configuration import conf_init, client_conf
+from zoe_client.state import init as state_init, create_tables
 
-argparser = None
-db_engine = None
 
+def get_zoe_client() -> ZoeClient:
+    return ZoeClient(client_conf().ipc_server, client_conf().ipc_port)
 
-def get_zoe_client(args) -> ZoeClient:
-    return ZoeClient(args.ipc_server, args.ipc_port)
 
-
-def stats_cmd(args):
-    client = get_zoe_client(args)
+def stats_cmd(_):
+    client = get_zoe_client()
     stats = client.platform_stats()
     pprint(stats)
 
 
 def user_new_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     user = client.user_new(args.email)
     print("New user ID: {}".format(user.id))
 
 
 def user_get_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     user = client.user_get_by_email(args.email)
     print("User ID: {}".format(user.id))
 
 
 def app_new_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     app_descr = json.load(args.jsonfile)
     application_id = client.application_new(args.user_id, app_descr)
     print("Application added with ID: {}".format(application_id))
 
 
 def app_bin_put_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     if not is_zipfile(args.zipfile):
         print("Error: application binary must be a zip file")
         return
@@ -52,7 +50,7 @@ def app_bin_put_cmd(args):
 
 
 def app_start_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     ret = client.application_start(args.id)
     if ret:
         print("Application scheduled successfully, use the app-inspect command to check its status")
@@ -61,12 +59,12 @@ def app_start_cmd(args):
 
 
 def app_rm_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     client.application_remove(args.id, args.force)
 
 
 def app_inspect_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     application = client.application_get(args.id)
     if application is None:
         print("Error: application {} does not exist".format(args.id))
@@ -75,7 +73,7 @@ def app_inspect_cmd(args):
 
 
 def app_list_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     applications = client.application_list(args.id)
     if len(applications) > 0:
         print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
@@ -84,7 +82,7 @@ def app_list_cmd(args):
 
 
 def exec_kill_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     execution = client.execution_get(args.id)
     if execution is None:
         print("Error: execution {} does not exist".format(args.id))
@@ -93,7 +91,7 @@ def exec_kill_cmd(args):
 
 
 def log_get_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     log = client.log_get(args.id)
     if log is None:
         print("Error: No log found for container ID {}".format(args.id))
@@ -101,22 +99,22 @@ def log_get_cmd(args):
 
 
 def gen_config_cmd(args):
-    zoeconf().write(open(args.output_file, "w"))
+    client_conf().write(open(args.output_file, "w"))
 
 
 def container_stats_cmd(args):
-    client = get_zoe_client(args)
+    client = get_zoe_client()
     stats = client.container_stats(args.container_id)
     print(stats)
 
 
 def process_arguments() -> Namespace:
-    global argparser
-    argparser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
-    argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
-    argparser.add_argument('--ipc-server', default='localhost', help='Address of the Zoe scheduler process')
-    argparser.add_argument('--ipc-port', default=8723, type=int, help='Port of the Zoe scheduler process')
-    subparser = argparser.add_subparsers(title='subcommands', description='valid subcommands')
+    parser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
+    parser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
+    parser.add_argument('--ipc-server', help='Address of the Zoe scheduler process')
+    parser.add_argument('--ipc-port', type=int, help='Port of the Zoe scheduler process')
+    parser.add_argument('--setup-db', action='store_true', help='Sets up the configured database for use with the Zoe client')
+    subparser = parser.add_subparsers(title='subcommands', description='valid subcommands')
 
     argparser_stats = subparser.add_parser('stats', help="Show the platform statistics")
     argparser_stats.set_defaults(func=stats_cmd)
@@ -172,21 +170,29 @@ def process_arguments() -> Namespace:
     argparser_container_stats.add_argument('container_id', help="ID of the container")
     argparser_container_stats.set_defaults(func=container_stats_cmd)
 
-    return argparser.parse_args()
+    return parser, parser.parse_args()
 
 
 def zoe():
-    args = process_arguments()
+    parser, args = process_arguments()
     if args.debug:
         logging.basicConfig(level=logging.DEBUG)
     else:
         logging.basicConfig(level=logging.INFO)
 
     conf_init()
-    state_init(zoeconf().db_url)
+    if hasattr(args, "ipc_server"):
+        client_conf().set('zoe_client', 'scheduler_ipc_address', args.ipc_server)
+    if hasattr(args, "ipc_port"):
+        client_conf().set('zoe_client', 'scheduler_ipc_port', args.ipc_port)
+
+    db_engine = state_init(client_conf().db_url)
+    if args.setup_db:
+        create_tables(db_engine)
+        sys.exit(0)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return
 
-#    try:
     args.func(args)
-#    except AttributeError:
-#        argparser.print_help()
-#        return

zoe_scheduler/configuration.py

+from configparser import ConfigParser
+
+config_paths = [
+    'zoe.conf',
+    'zoe-scheduler.conf',
+    '/etc/zoe/zoe.conf',
+    '/etc/zoe/zoe-scheduler.conf'
+]
+
+defaults = {
+    'zoe_scheduler': {
+        'swarm_manager_url': 'tcp://swarm.example.com:2380',
+        'docker_private_registry': '10.1.0.1:5000',
+        'status_refresh_interval': 10,
+        'check_terminated_interval': 30,
+        'db_connect': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe',
+        'storage_path': "/var/lib/zoe/history",
+        'http_listen_address': '127.0.0.1',
+        'http_listen_port': 4390,
+        'ipc_listen_address': '127.0.0.1',
+        'ipc_listen_port': 8723
+    }
+}
+
+_zoeconf = None
+
+
+class ZoeSchedulerConfig(ConfigParser):
+    def __init__(self):
+        super().__init__(interpolation=None)
+        self.read_dict(defaults)
+
+    @staticmethod
+    def write_defaults(cls, fp):
+        tmp = cls()
+        tmp.write(fp)
+
+    @property
+    def storage_path(self) -> str:
+        return self.get('zoe_scheduler', 'storage_path')
+
+    @property
+    def check_terminated_interval(self) -> int:
+        return self.getint('zoe_scheduler', 'check_terminated_interval')
+
+    @property
+    def db_url(self) -> str:
+        return self.get('zoe_scheduler', 'db_connect')
+
+    @property
+    def status_refresh_interval(self) -> int:
+        return self.getint('zoe_scheduler', 'status_refresh_interval')
+
+    @property
+    def docker_swarm_manager(self) -> str:
+        return self.get('zoe_scheduler', 'swarm_manager_url')
+
+    @property
+    def docker_private_registry(self) -> str:
+        return self.get('zoe_scheduler', 'docker_private_registry')
+
+    @property
+    def http_listen_port(self) -> int:
+        return self.getint('zoe_scheduler', 'http_listen_port')
+
+    @property
+    def http_listen_address(self) -> str:
+        return self.get('zoe_scheduler', 'http_listen_address')
+
+    @property
+    def ipc_listen_port(self) -> int:
+        return self.getint('zoe_scheduler', 'ipc_listen_port')
+
+    @property