Commit eb8d2f71 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Refactor configuration

Split configuration into different modules for the scheduler and the clients
parent 47ed78a6
......@@ -59,4 +59,5 @@ docs/_build/
# PyBuilder
target/
.idea/
zoe.conf
zoe-scheduler.conf
zoe-client.conf
from configparser import ConfigParser
ipcconf = {
'server': None,
'port': None,
}
config_paths = [
'zoe.conf',
'/etc/zoe/zoe.conf'
]
defaults = {
'docker': {
'swarm_manager_url': 'tcp://swarm.example.com:2380',
'private_registry': '10.1.0.1:5000'
},
'intervals': {
'status_refresh': 10,
'scheduler_task': 10,
'check_health': 30,
'notebook_max_age_no_activity': 24,
'notebook_warning_age_no_activity': 2
},
'db': {
'url': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe'
},
'smtp': {
'server': 'smtp.exmaple.com',
'user': 'zoe@exmaple.com',
'password': 'changeme'
},
'filesystem': {
'history_path': "/var/lib/zoe/history"
},
'flask': {
'secret_key': b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
},
'network': {
'scheduler_internal_server_port': 4390,
'scheduler_internal_hostname': '127.0.0.1'
}
}
_zoeconf = None
class ZoeConfig(ConfigParser):
def __init__(self):
super().__init__(interpolation=None)
self.read_dict(defaults)
def write_defaults(self, fp):
tmp = ZoeConfig()
tmp.write(fp)
@property
def history_path(self) -> str:
return self.get('filesystem', 'history_path')
@property
def web_server_name(self) -> str:
return self.get('apache', 'web_server_name')
@property
def smtp_server(self) -> str:
return self.get('smtp', 'server')
@property
def smtp_user(self) -> str:
return self.get('smtp', 'user')
@property
def smtp_password(self) -> str:
return self.get('smtp', 'password')
@property
def notebook_warning_age_no_activity(self) -> int:
return self.getint('intervals', 'notebook_warning_age_no_activity')
@property
def notebook_max_age_no_activity(self) -> int:
return self.getint('intervals', 'notebook_max_age_no_activity')
@property
def interval_check_health(self) -> int:
return self.getint('intervals', 'check_health')
@property
def db_url(self) -> str:
return self.get('db', 'url')
@property
def interval_scheduler_task(self) -> int:
return self.getint('intervals', 'scheduler_task')
@property
def interval_status_refresh(self) -> int:
return self.getint('intervals', 'status_refresh')
@property
def docker_swarm_manager(self) -> str:
return self.get('docker', 'swarm_manager_url')
@property
def cookies_secret_key(self):
return self.get('flask', 'secret_key')
@property
def docker_private_registry(self) -> str:
return self.get('docker', 'private_registry')
@property
def scheduler_internal_server_port(self) -> int:
return self.getint('network', 'scheduler_internal_server_port')
@property
def scheduler_internal_hostname(self) -> str:
return self.get('network', 'scheduler_internal_hostname')
def init(config_file=None) -> ZoeConfig:
global _zoeconf
_zoeconf = ZoeConfig()
if config_file is None:
_zoeconf.read(config_paths)
else:
_zoeconf.read_file(open(config_file))
return _zoeconf
def zoeconf() -> ZoeConfig:
return _zoeconf
import json
import pytest
from zoe_scheduler.state import init as state_init, Base, AlchemySession
from zoe_scheduler.state.application import ApplicationState
from zoe_scheduler.application_description import ZoeApplication
from common.configuration import init as conf_init, zoeconf
from zoe_scheduler.configuration import init as conf_init, scheduler_conf
def pytest_addoption(parser):
......@@ -27,7 +27,7 @@ def configuration(environment):
@pytest.fixture(scope='session')
def state_connection(request, configuration):
engine = state_init(zoeconf().db_url)
engine = state_init(scheduler_conf().db_url)
connection = engine.connect()
trans = connection.begin()
......
[docker]
[zoe_scheduler]
swarm_manager_url = tcp://example.com:2380
private_registry = 10.1.0.1:5000
docker_private_registry = 10.1.0.1:5000
status_refresh_interval = 10
check_terminated_interval = 30
db_connect = mysql+mysqlconnector://root@localhost/zoe
storage_path = /tmp/zoe
http_listen_address = 192.168.45.25
http_listen_port = 4390
ipc_listen_address = 127.0.0.1
ipc_listen_port = 8723
[intervals]
check_health = 30
notebook_max_age_no_activity = 24
scheduler_task = 10
notebook_warning_age_no_activity = 2
status_refresh = 10
[zoe_client]
db_connect = mysql+mysqlconnector://root@localhost/zoe
scheduler_ipc_address = localhost
scheduler_ipc_port = 8723
[filesystem]
history_path = /tmp/history
[flask]
secret_key = b"\xc4\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
[smtp]
password = none
user = none
server = none
[db]
url = mysql+mysqlconnector://root@localhost/zoe
[network]
scheduler_internal_server_port = 4390
scheduler_internal_hostname = 192.168.45.25
[zoe_web]
smtp_password = pass
smtp_user = user
smtp_server = smtp.example.com
cookie_secret = \xc3\xb0\xa7\xaa\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
web_server_name = localhost
[docker]
[zoe_scheduler]
swarm_manager_url = tcp://example.com:2380
docker_private_registry = 10.1.0.1:5000
status_refresh_interval = 10
check_terminated_interval = 30
db_connect = mysql+mysqlconnector://root@localhost/zoe
storage_path = /tmp/zoe
http_listen_address = 192.168.45.25
http_listen_port = 4390
ipc_listen_address = 127.0.0.1
ipc_listen_port = 8723
[intervals]
check_health = 30
notebook_max_age_no_activity = 24
scheduler_task = 10
notebook_warning_age_no_activity = 2
status_refresh = 10
[zoe_client]
db_connect = mysql+mysqlconnector://root@localhost/zoe
scheduler_ipc_address = localhost
scheduler_ipc_port = 8723
[filesystem]
history_path = /tmp/history
[flask]
secret_key = b"\xc4\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!"
[smtp]
password = none
user = none
server = none
[db]
url = mysql+mysqlconnector://root@localhost/zoe
[zoe_web]
smtp_password = pass
smtp_user = user
smtp_server = smtp.example.com
cookie_secret = \xc3\xb0\xa7\xaa\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
web_server_name = localhost
[zoe_client]
db_connect = mysql+mysqlconnector://root@localhost/zoe
scheduler_ipc_address = localhost
scheduler_ipc_port = 8723
[zoe_web]
smtp_password = Daicu2Ze
smtp_user = bigfoot.data@gmail.com
smtp_server = smtp.gmail.com
cookie_secret = \xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!
web_server_name = 192.168.45.25
......@@ -5,7 +5,6 @@ from sqlalchemy.orm.exc import NoResultFound
from zoe_client.state import AlchemySession
from zoe_client.ipc import ZoeIPCClient
from common.configuration import zoeconf
from zoe_client.entities import Execution, Application, User
from zoe_client.state.user import UserState
......@@ -15,7 +14,6 @@ log = logging.getLogger(__name__)
class ZoeClient:
def __init__(self, ipc_server='localhost', ipc_port=8723):
self.ipc_server = ZoeIPCClient(ipc_server, ipc_port)
self.image_registry = zoeconf().docker_private_registry
self.state = AlchemySession()
# Applications
......
from configparser import ConfigParser
config_paths = [
'zoe.conf',
'zoe-client.conf',
'/etc/zoe/zoe.conf',
'/etc/zoe/zoe-client.conf'
]
defaults = {
'zoe_client': {
'db_connect': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe',
'scheduler_ipc_address': 'localhost',
'scheduler_ipc_port': 8723
},
'zoe_web': {
'smtp_server': 'smtp.exmaple.com',
'smtp_user': 'zoe@exmaple.com',
'smtp_password': 'changeme',
'cookie_secret': b"\xc3\xb0\xa7\xff\x8fH'\xf7m\x1c\xa2\x92F\x1d\xdcz\x05\xe6CJN5\x83!",
'web_server_name': 'localhost'
}
}
_zoeconf = None
class ZoeClientConfig(ConfigParser):
def __init__(self):
super().__init__(interpolation=None)
self.read_dict(defaults)
@staticmethod
def write_defaults(cls, fp):
tmp = cls()
tmp.write(fp)
@property
def db_url(self) -> str:
return self.get('zoe_client', 'db_connect')
@property
def ipc_server(self) -> str:
return self.get('zoe_client', 'scheduler_ipc_address')
@property
def ipc_port(self) -> int:
return self.getint('zoe_client', 'scheduler_ipc_port')
@property
def web_server_name(self) -> str:
return self.get('zoe_web', 'web_server_name')
@property
def smtp_server(self) -> str:
return self.get('zoe_web', 'smtp_server')
@property
def smtp_user(self) -> str:
return self.get('zoe_web', 'smtp_user')
@property
def smtp_password(self) -> str:
return self.get('zoe_web', 'smtp_password')
@property
def cookies_secret_key(self):
return self.get('zoe_web', 'cookie_secret')
def conf_init(config_file=None) -> ZoeClientConfig:
global _zoeconf
_zoeconf = ZoeClientConfig()
if config_file is None:
_zoeconf.read(config_paths)
else:
_zoeconf.read_file(open(config_file))
return _zoeconf
def client_conf() -> ZoeClientConfig:
return _zoeconf
......@@ -3,46 +3,44 @@ import json
import logging
from zipfile import is_zipfile
from pprint import pprint
import sys
from zoe_client import ZoeClient
from common.configuration import init as conf_init, zoeconf
from zoe_client.state import create_tables, init as state_init
from zoe_client.configuration import conf_init, client_conf
from zoe_client.state import init as state_init, create_tables
argparser = None
db_engine = None
def get_zoe_client() -> ZoeClient:
return ZoeClient(client_conf().ipc_server, client_conf().ipc_port)
def get_zoe_client(args) -> ZoeClient:
return ZoeClient(args.ipc_server, args.ipc_port)
def stats_cmd(args):
client = get_zoe_client(args)
def stats_cmd(_):
client = get_zoe_client()
stats = client.platform_stats()
pprint(stats)
def user_new_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
user = client.user_new(args.email)
print("New user ID: {}".format(user.id))
def user_get_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
user = client.user_get_by_email(args.email)
print("User ID: {}".format(user.id))
def app_new_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
app_descr = json.load(args.jsonfile)
application_id = client.application_new(args.user_id, app_descr)
print("Application added with ID: {}".format(application_id))
def app_bin_put_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
if not is_zipfile(args.zipfile):
print("Error: application binary must be a zip file")
return
......@@ -52,7 +50,7 @@ def app_bin_put_cmd(args):
def app_start_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
ret = client.application_start(args.id)
if ret:
print("Application scheduled successfully, use the app-inspect command to check its status")
......@@ -61,12 +59,12 @@ def app_start_cmd(args):
def app_rm_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
client.application_remove(args.id, args.force)
def app_inspect_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
application = client.application_get(args.id)
if application is None:
print("Error: application {} does not exist".format(args.id))
......@@ -75,7 +73,7 @@ def app_inspect_cmd(args):
def app_list_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
applications = client.application_list(args.id)
if len(applications) > 0:
print("{:4} {:20} {:25}".format("ID", "Name", "Type"))
......@@ -84,7 +82,7 @@ def app_list_cmd(args):
def exec_kill_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
execution = client.execution_get(args.id)
if execution is None:
print("Error: execution {} does not exist".format(args.id))
......@@ -93,7 +91,7 @@ def exec_kill_cmd(args):
def log_get_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
log = client.log_get(args.id)
if log is None:
print("Error: No log found for container ID {}".format(args.id))
......@@ -101,22 +99,22 @@ def log_get_cmd(args):
def gen_config_cmd(args):
zoeconf().write(open(args.output_file, "w"))
client_conf().write(open(args.output_file, "w"))
def container_stats_cmd(args):
client = get_zoe_client(args)
client = get_zoe_client()
stats = client.container_stats(args.container_id)
print(stats)
def process_arguments() -> Namespace:
global argparser
argparser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
argparser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
argparser.add_argument('--ipc-server', default='localhost', help='Address of the Zoe scheduler process')
argparser.add_argument('--ipc-port', default=8723, type=int, help='Port of the Zoe scheduler process')
subparser = argparser.add_subparsers(title='subcommands', description='valid subcommands')
parser = ArgumentParser(description="Zoe - Container Analytics as a Service command-line client")
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Enable debug output')
parser.add_argument('--ipc-server', help='Address of the Zoe scheduler process')
parser.add_argument('--ipc-port', type=int, help='Port of the Zoe scheduler process')
parser.add_argument('--setup-db', action='store_true', help='Sets up the configured database for use with the Zoe client')
subparser = parser.add_subparsers(title='subcommands', description='valid subcommands')
argparser_stats = subparser.add_parser('stats', help="Show the platform statistics")
argparser_stats.set_defaults(func=stats_cmd)
......@@ -172,21 +170,29 @@ def process_arguments() -> Namespace:
argparser_container_stats.add_argument('container_id', help="ID of the container")
argparser_container_stats.set_defaults(func=container_stats_cmd)
return argparser.parse_args()
return parser, parser.parse_args()
def zoe():
args = process_arguments()
parser, args = process_arguments()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
conf_init()
state_init(zoeconf().db_url)
if hasattr(args, "ipc_server"):
client_conf().set('zoe_client', 'scheduler_ipc_address', args.ipc_server)
if hasattr(args, "ipc_port"):
client_conf().set('zoe_client', 'scheduler_ipc_port', args.ipc_port)
db_engine = state_init(client_conf().db_url)
if args.setup_db:
create_tables(db_engine)
sys.exit(0)
if not hasattr(args, "func"):
parser.print_help()
return
# try:
args.func(args)
# except AttributeError:
# argparser.print_help()
# return
from configparser import ConfigParser
config_paths = [
'zoe.conf',
'zoe-scheduler.conf',
'/etc/zoe/zoe.conf',
'/etc/zoe/zoe-scheduler.conf'
]
defaults = {
'zoe_scheduler': {
'swarm_manager_url': 'tcp://swarm.example.com:2380',
'docker_private_registry': '10.1.0.1:5000',
'status_refresh_interval': 10,
'check_terminated_interval': 30,
'db_connect': 'mysql+mysqlconnector://zoe:pass@dbhost/zoe',
'storage_path': "/var/lib/zoe/history",
'http_listen_address': '127.0.0.1',
'http_listen_port': 4390,
'ipc_listen_address': '127.0.0.1',
'ipc_listen_port': 8723
}
}
_zoeconf = None
class ZoeSchedulerConfig(ConfigParser):
def __init__(self):
super().__init__(interpolation=None)
self.read_dict(defaults)
@staticmethod
def write_defaults(cls, fp):
tmp = cls()
tmp.write(fp)
@property
def storage_path(self) -> str:
return self.get('zoe_scheduler', 'storage_path')
@property
def check_terminated_interval(self) -> int:
return self.getint('zoe_scheduler', 'check_terminated_interval')
@property
def db_url(self) -> str:
return self.get('zoe_scheduler', 'db_connect')
@property
def status_refresh_interval(self) -> int:
return self.getint('zoe_scheduler', 'status_refresh_interval')
@property
def docker_swarm_manager(self) -> str:
return self.get('zoe_scheduler', 'swarm_manager_url')
@property
def docker_private_registry(self) -> str:
return self.get('zoe_scheduler', 'docker_private_registry')
@property
def http_listen_port(self) -> int:
return self.getint('zoe_scheduler', 'http_listen_port')
@property
def http_listen_address(self) -> str:
return self.get('zoe_scheduler', 'http_listen_address')
@property
def ipc_listen_port(self) -> int:
return self.getint('zoe_scheduler', 'ipc_listen_port')
@property