Commit 8006ee3d authored by Daniele Venzano's avatar Daniele Venzano

Move the swarm backend in a package and abstract the interface

parent fcb279b2
......@@ -190,11 +190,11 @@ Will return a JSON document like this::
{
"status" : "active",
"service_group" : "boinc-client",
"docker_status" : "started",
"backend_status" : "started",
"ip_address" : "10.0.0.94",
"execution_id" : 25158,
"name" : "boinc-client0",
"docker_id" : "d0042c69b54e90327d9287e099304b6c25921d81f639803494ea744445d58430",
"backend_id" : "d0042c69b54e90327d9287e099304b6c25921d81f639803494ea744445d58430",
"error_message" : null,
"id" : 26774,
"description" : {
......@@ -211,11 +211,11 @@ Where:
* ``status`` is the service status from Zoe point of view. It can be one of "terminating", "inactive", "active" or "starting"
* ``service_group`` is the name for the service provided in the ZApp description. When the ZApp is unpacked to create the actual containers a single service definition will spawn one or more services with this name in common
* ``docker_status`` is the container status from the point of view of Docker. Zoe tries her best to keep this value in sync, but the value here can be out of sync by several minutes. It can be one of 'undefined', 'created', 'started', 'dead' or 'destroyed'
* ``backend_status`` is the container status from the point of view of the container backend. Zoe tries her best to keep this value in sync, but the value here can be out of sync by several minutes. It can be one of 'undefined', 'created', 'started', 'dead' or 'destroyed'
* ``ip_address`` is the IP address of the container
* ``execution_id`` is the execution ID this service belongs to
* ``name`` is the name for this service instance, generated from ``service_group``
* ``docker_id`` is the Docker ID string
* ``backend_id`` is the ID used by the backend to identify this container
* ``error_message`` is currently unused
* ``id`` is the ID of this service, should match the one given in the URL
* ``description`` is the service description extracted from the ZApp
......
#!/usr/bin/env python3
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Bypass the Zoe scheduler to run a ZApp and leave the logs inside Docker, used for debugging ZApps."""
import datetime
import json
import logging
import time
import zoe_lib.applications
import zoe_lib.config as config
from zoe_lib.configargparse import ArgumentParser, FileType
from zoe_lib.state.sql_manager import Execution, Service
from zoe_lib.swarm_client import SwarmClient
from zoe_master.backends.old_swarm import execution_to_containers, terminate_execution
from zoe_master.execution_manager import _digest_application_description
log = logging.getLogger("main")
LOG_FORMAT = '%(asctime)-15s %(levelname)s %(name)s (%(threadName)s): %(message)s'
CONFIG_PATHS = [
'zoe.conf',
'/etc/zoe/zoe.conf'
]
class FakeSQLManager:
"""A fake in-memory state class."""
def __init__(self):
self.executions = []
self.services = []
self._last_id = 0
def execution_list(self, only_one=False, **kwargs):
"""Execution list."""
ret_list = []
for e in self.executions:
for key, value in kwargs.items():
if getattr(e, key) == value:
ret_list.append(e)
if only_one:
return ret_list[0]
else:
return ret_list
def execution_update(self, exec_id, **kwargs):
"""Execution update."""
for e in self.executions:
if e.id == exec_id:
for key, value in kwargs.items():
if key == "status":
continue
print(key, value)
setattr(e, key, value)
break
def execution_new(self, name, user_id, description):
"""New execution."""
e_dict = {
'id': self._last_id,
'name': name,
'user_id': user_id,
'description': description,
'status': Execution.SUBMIT_STATUS,
'time_submit': datetime.datetime.now(),
'time_start': None,
'time_end': None,
'error_message': None
}
e = Execution(e_dict, self)
self.executions.append(e)
self._last_id += 1
return self._last_id - 1
def execution_delete(self, execution_id):
"""Delete execution."""
raise NotImplementedError
def service_list(self, only_one=False, **kwargs):
"""Service list."""
ret_list = []
for e in self.services:
for key, value in kwargs.items():
if getattr(e, key) == value:
ret_list.append(e)
if only_one:
return ret_list[0]
else:
return ret_list
def service_update(self, service_id, **kwargs):
"""Service update."""
for e in self.services:
if e.id == service_id:
for key, value in kwargs.items():
setattr(e, key, value)
break
def service_new(self, execution_id, name, service_group, description):
"""Service new."""
s_dict = {
'id': self._last_id,
'name': name,
'description': description,
'status': Execution.SUBMIT_STATUS,
'execution_id': execution_id,
'docker_id': None,
'service_group': service_group,
'error_message': None,
'docker_status': Service.BACKEND_UNDEFINED_STATUS
}
service = Service(s_dict, self)
self.services.append(service)
self._last_id += 1
return self._last_id - 1
def load_configuration():
"""Load configuration from the command line."""
argparser = ArgumentParser(description="Zoe application tester - Container Analytics as a Service core component",
default_config_files=CONFIG_PATHS,
auto_env_var_prefix="ZOE_MASTER_",
args_for_setting_config_path=["--config"],
args_for_writing_out_config_file=["--write-config"])
argparser.add_argument('--debug', action='store_true', help='Enable debug output')
argparser.add_argument('--swarm', help='Swarm/Docker API endpoint (ex.: zk://zk1:2181,zk2:2181 or http://swarm:2380)', default='http://localhost:2375')
argparser.add_argument('--deployment-name', help='name of this Zoe deployment', default='prod')
argparser.add_argument('--dbname', help='DB name', default='zoe')
argparser.add_argument('--dbuser', help='DB user', default='zoe')
argparser.add_argument('--dbpass', help='DB password', default='')
argparser.add_argument('--dbhost', help='DB hostname', default='localhost')
argparser.add_argument('--dbport', type=int, help='DB port', default=5432)
# Master options
argparser.add_argument('--api-listen-uri', help='ZMQ API listen address', default='tcp://*:4850')
argparser.add_argument('--influxdb-dbname', help='Name of the InfluxDB database to use for storing metrics', default='zoe')
argparser.add_argument('--influxdb-url', help='URL of the InfluxDB service (ex. http://localhost:8086)', default='http://localhost:8086')
argparser.add_argument('--influxdb-enable', action="store_true", help='Enable metric output toward influxDB')
argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:1234)', default='')
argparser.add_argument('--workspace-base-path', help='Path where user workspaces will be created by Zoe. Must be visible at this path on all Swarm hosts.', default='/mnt/zoe-workspaces')
argparser.add_argument('--overlay-network-name', help='Name of the Swarm overlay network Zoe should use', default='zoe')
# API options
argparser.add_argument('--listen-address', type=str, help='Address to listen to for incoming connections', default="0.0.0.0")
argparser.add_argument('--listen-port', type=int, help='Port to listen to for incoming connections', default=5001)
argparser.add_argument('--master-url', help='URL of the Zoe master process', default='tcp://127.0.0.1:4850')
# API auth options
argparser.add_argument('--auth-type', help='Authentication type (text or ldap)', default='text')
argparser.add_argument('--auth-file', help='Path to the CSV file containing user,pass,role lines for text authentication', default='zoepass.csv')
argparser.add_argument('--ldap-server-uri', help='LDAP server to use for authentication', default='ldap://localhost')
argparser.add_argument('--ldap-base-dn', help='LDAP base DN for users', default='ou=something,dc=any,dc=local')
argparser.add_argument('--ldap-admin-gid', type=int, help='LDAP group ID for admins', default=5000)
argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
argparser.add_argument('jsonfile', type=FileType("r"), help='Application description')
opts = argparser.parse_args()
opts.gelf_address = '' # For debugging we want to easily look at logs with 'docker logs'
opts.influxdb_enable = False # don't send metrics for these test runs
opts.deployment_name = 'zapp-test'
if opts.debug:
argparser.print_values()
return opts
def main():
"""The main entrypoint function."""
conf = load_configuration()
config.load_configuration(conf)
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logging.getLogger('kazoo').setLevel(logging.WARNING)
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('docker').setLevel(logging.INFO)
logging.getLogger("tornado").setLevel(logging.DEBUG)
state = FakeSQLManager()
zapp_description = json.load(args.jsonfile)
print('Validating zapp description...')
zoe_lib.applications.app_validate(zapp_description)
exec_id = state.execution_new('test', 'fake_user', zapp_description)
e = state.execution_list(only_one=True, id=exec_id)
_digest_application_description(state, e)
print('Zapp digested, starting containers...')
execution_to_containers(e)
print('Giving the containers a few seconds to start...')
time.sleep(5)
swarm = SwarmClient(args)
for service in e.services:
print("Service {}, docker ID: {}".format(service.name, service.docker_id))
logs = swarm.logs(service.docker_id, False)
logs = logs.decode('utf-8').split('\n')
for log_line in logs[-10:]:
print(log_line)
print("Execution as been started, press CTRL-C to terminate it")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
print('Terminating...')
terminate_execution(e)
if __name__ == '__main__':
main()
......@@ -24,7 +24,6 @@ import zoe_lib.applications
import zoe_lib.exceptions
import zoe_lib.state
from zoe_lib.config import get_conf
from zoe_lib.swarm_client import SwarmClient
log = logging.getLogger(__name__)
......@@ -124,18 +123,6 @@ class APIEndpoint:
ret = [s for s in services if s.user_id == uid or role == 'admin']
return ret
def service_logs(self, uid, role, service_id, stream=True):
"""Retrieve the logs for the given service."""
service = self.sql.service_list(id=service_id, only_one=True)
if service is None:
raise zoe_api.exceptions.ZoeNotFoundException('No such service')
if service.user_id != uid and role != 'admin':
raise zoe_api.exceptions.ZoeAuthException()
if service.docker_id is None:
raise zoe_api.exceptions.ZoeNotFoundException('Container is not running')
swarm = SwarmClient(get_conf())
return swarm.logs(service.docker_id, stream)
def statistics_scheduler(self, uid_, role_):
"""Retrieve statistics about the scheduler."""
success, message = self.master.scheduler_statistics()
......@@ -159,7 +146,7 @@ class APIEndpoint:
for execution in all_execs:
if execution.status == execution.RUNNING_STATUS:
for service in execution.services:
if service.description['monitor'] and service.docker_status == service.DOCKER_DIE_STATUS or service.docker_status == service.DOCKER_DESTROY_STATUS:
if service.description['monitor'] and service.backend_status == service.BACKEND_DIE_STATUS or service.backend_status == service.BACKEND_DESTROY_STATUS:
log.info("Service {} of execution {} died, terminating execution".format(service.name, execution.id))
self.master.execution_terminate(execution.id)
break
......
......@@ -73,8 +73,9 @@ def create_tables(cur):
execution_id INT REFERENCES execution,
service_group TEXT NOT NULL,
name TEXT NOT NULL,
docker_id TEXT NULL DEFAULT NULL,
docker_status TEXT NOT NULL DEFAULT 'undefined'
backend_id TEXT NULL DEFAULT NULL,
backend_status TEXT NOT NULL DEFAULT 'undefined',
ip_address CIDR NULL DEFAULT NULL
)''')
......
......@@ -21,7 +21,7 @@ import tornado.web
from zoe_api.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI, ExecutionDeleteAPI
from zoe_api.rest_api.info import InfoAPI
from zoe_api.rest_api.service import ServiceAPI, ServiceLogsAPI
from zoe_api.rest_api.service import ServiceAPI
from zoe_api.rest_api.discovery import DiscoveryAPI
from zoe_api.rest_api.statistics import SchedulerStatsAPI
......@@ -44,7 +44,6 @@ def api_init(api_endpoint) -> List[tornado.web.URLSpec]:
tornado.web.url(API_PATH + r'/execution', ExecutionCollectionAPI, route_args),
tornado.web.url(API_PATH + r'/service/([0-9]+)', ServiceAPI, route_args),
tornado.web.url(API_PATH + r'/service/logs/([0-9]+)', ServiceLogsAPI, route_args),
tornado.web.url(API_PATH + r'/discovery/by_group/([0-9]+)/([a-z0-9A-Z\-]+)', DiscoveryAPI, route_args),
......
......@@ -49,48 +49,3 @@ class ServiceAPI(RequestHandler):
def data_received(self, chunk):
"""Not implemented as we do not use stream uploads"""
pass
class ServiceLogsAPI(RequestHandler):
"""The Service logs API endpoint."""
def initialize(self, **kwargs):
"""Initializes the request handler."""
self.api_endpoint = kwargs['api_endpoint'] # type: APIEndpoint
self.connection_closed = False
def on_connection_close(self):
"""Tornado callback for clients closing the connection."""
self.connection_closed = True
@catch_exceptions
@tornado.gen.coroutine
def get(self, service_id):
"""HTTP GET method."""
uid, role = get_auth(self)
log_gen = self.api_endpoint.service_logs(uid, role, service_id, stream=True)
while True:
try:
log_line = yield THREAD_POOL.submit(next, log_gen)
except StopIteration:
break
self.write(log_line)
try:
yield self.flush()
except tornado.iostream.StreamClosedError:
break
if self.connection_closed:
break
log.debug('Finished log stream for service {}'.format(service_id))
self.finish()
def data_received(self, chunk):
"""Not implemented as we do not use stream uploads"""
pass
......@@ -157,10 +157,10 @@ def exec_get_cmd(args):
service = cont_api.get(c_id)
print('Service {} (ID: {})'.format(service['name'], service['id']))
print(' - zoe status: {}'.format(service['status']))
print(' - docker status: {}'.format(service['docker_status']))
print(' - backend status: {}'.format(service['backend_status']))
if service['error_message'] is not None:
print(' - error: {}'.format(service['error_message']))
if service['docker_status'] == 'started':
if service['backend_status'] == 'started':
ip = service['ip_address']
for port in service['description']['ports']:
print(' - {}: {}://{}:{}{}'.format(port['name'], port['protocol'], ip, port['port_number'], port['path']))
......
......@@ -67,12 +67,6 @@ def app_validate(data):
if 'services' not in data:
raise InvalidApplicationDescription(msg='the application should contain a list of services')
if 'disable_autorestart' in data:
try:
bool(data['disable_autorestart'])
except ValueError:
raise InvalidApplicationDescription(msg="disable_autorestart field should be a boolean")
for service in data['services']:
_service_check(service)
......@@ -122,11 +116,17 @@ def _service_check(data):
if not isinstance(data['required_resources'], dict):
raise InvalidApplicationDescription(msg="required_resources should be a dictionary")
if 'memory' not in data['required_resources']:
raise InvalidApplicationDescription(msg="Missing required key: required_resources -> memory")
data['required_resources']['memory'] = 0
if 'cores' not in data['required_resources']:
data['required_resources']['cores'] = 0
try:
int(data['required_resources']['memory'])
except ValueError:
raise InvalidApplicationDescription(msg="required_resources -> memory field should be an int")
try:
int(data['required_resources']['cores'])
except ValueError:
raise InvalidApplicationDescription(msg="required_resources -> cores field should be an int")
if 'environment' in data:
if not hasattr(data['environment'], '__iter__'):
......
......@@ -84,10 +84,10 @@ def load_configuration(test_conf=None):
argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
argparser.add_argument('--service-log-path', help='Save service logs in this directory, EXPERIMENTAL', default='')
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', default='ZoeSimpleScheduler')
argparser.add_argument('--backend', choices=['OldSwarm'], default='OldSwarm')
opts = argparser.parse_args()
if opts.debug:
argparser.print_values()
......
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Container log storage"""
import logging
import os
import shutil
from zoe_lib.config import get_conf
from zoe_lib.state import Execution
from zoe_lib.swarm_client import SwarmClient
log = logging.getLogger(__name__)
def _path_from_execution(execution: Execution):
return os.path.join(get_conf().service_log_path, get_conf().deployment_name, str(execution.id))
def _init(execution: Execution):
if get_conf().service_log_path == '':
return None
base_path = _path_from_execution(execution)
try:
os.makedirs(base_path, exist_ok=True)
except (OSError, PermissionError):
log.exception('Error creating the directory at path: {}'.format(base_path))
return None
return base_path
def _shutdown():
pass
def save(execution: Execution):
"""Save the logs of the service specified as argument"""
path = _init(execution)
if path is None:
return
for service in execution.services:
fname = service.name + '.txt'
fpath = os.path.join(path, fname)
swarm = SwarmClient(get_conf())
log_gen = swarm.logs(service.docker_id, stream=True, follow=False)
if log_gen is None:
_shutdown()
return
try:
with open(fpath, 'wb') as out_fp:
for line in log_gen:
out_fp.write(line)
except FileNotFoundError:
log.error("Could not create file {}".format(fpath))
_shutdown()
def delete(execution: Execution):
"""Delete the logs for a service"""
path = _init(execution)
if path is None:
return
shutil.rmtree(path, ignore_errors=True)
_shutdown()
......@@ -18,7 +18,7 @@
import logging
from zoe_lib.config import get_conf
from zoe_lib.swarm_client import SwarmClient
from zoe_master.backends.old_swarm.api_client import SwarmClient
log = logging.getLogger(__name__)
......@@ -76,6 +76,10 @@ class Service:
self.backend_id = d['backend_id']
self.backend_status = d['backend_status']
self.ip_address = d['ip_address']
if self.ip_address is not None and ('/32' in self.ip_address or '/128' in self.ip_address):
self.ip_address = self.ip_address.split('/')[0]
# Fields parsed from the JSON description
self.image_name = self.description['docker_image']
self.is_monitor = self.description['monitor']
......@@ -110,11 +114,6 @@ class Service:
def __eq__(self, other):
return self.id == other.id
@property
def dns_name(self):
"""Getter for the DNS name of this service as it will be registered in Docker's DNS."""
return "{}-{}-{}".format(self.name, self.execution_id, get_conf().deployment_name)
def set_terminating(self):
"""The service is being killed."""
self.sql_manager.service_update(self.id, status=self.TERMINATING_STATUS)
......@@ -122,7 +121,7 @@ class Service:
def set_inactive(self):
"""The service is not running."""
self.sql_manager.service_update(self.id, status=self.INACTIVE_STATUS, docker_id=None)
self.sql_manager.service_update(self.id, status=self.INACTIVE_STATUS, backend_id=None, ip_address=None)
self.status = self.INACTIVE_STATUS
def set_starting(self):
......@@ -130,10 +129,12 @@ class Service:
self.sql_manager.service_update(self.id, status=self.STARTING_STATUS)
self.status = self.STARTING_STATUS
def set_active(self, docker_id):
"""The service is running and has a valid docker_id."""
self.sql_manager.service_update(self.id, status=self.ACTIVE_STATUS, docker_id=docker_id, error_message=None)
def set_active(self, backend_id, ip_address):
"""The service is running and has a valid backend_id."""
self.sql_manager.service_update(self.id, status=self.ACTIVE_STATUS, backend_id=backend_id, error_message=None, ip_address=ip_address)
self.error_message = None
self.ip_address = ip_address
self.backend_id = backend_id
self.status = self.ACTIVE_STATUS
def set_error(self, error_message):
......@@ -146,16 +147,12 @@ class Service:
"""Docker has emitted an event related to this service."""
self.sql_manager.service_update(self.id, backend_status=new_status)
log.debug("service {}, backend status updated to {}".format(self.id, new_status))
self.docker_status = new_status
self.backend_status = new_status
@property
def ip_address(self):
"""Getter for the service IP address, queries Swarm as the IP address changes outside our control."""
if self.docker_status != self.BACKEND_START_STATUS:
return {}
swarm = SwarmClient(get_conf())
s_info = swarm.inspect_container(self.backend_id)
return s_info['ip_address'][get_conf().overlay_network_name]
def dns_name(self):
"""Getter for the DNS name of this service as it will be registered in Docker's DNS."""
return "{}-{}-{}".format(self.name, self.execution_id, get_conf().deployment_name)
@property
def user_id(self):
......
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The base class that all backends should implement."""
from typing import Dict
from zoe_lib.state import Execution, Service
class BaseBackend:
"""The base class that all backends should implement."""
def __init__(self, conf):
pass
def init(self, state):
"""Initializes Swarm backend starting the event monitoring thread."""
raise NotImplementedError
def shutdown(self):
"""Performs a clean shutdown of the resources used by Swarm backend."""
raise NotImplementedError
def spawn_service(self, execution: Execution, service: Service, env_subst_dict: Dict):
raise NotImplementedError
def terminate_service(self, service: Service) -> None:
raise NotImplementedError
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The high-level interface that Zoe uses to talk to the configured container backend."""
from typing import Dict
from zoe_lib.state import Service
from zoe_master.exceptions import ZoeStartExecutionFatalException
def gen_environment(service: Service, env_subst_dict: Dict):
""" Generate a dictionary containing the current cluster status (before the new container is spawned)
This information is used to substitute template strings in the environment variables."""
env_list = []
for env_name, env_value in service.environment:
try:
env_value = env_value.format(**env_subst_dict)
except KeyError:
error_msg = "Unknown variable in environment expression '{}', known variables are: {}".format(env_value, list(env_subst_dict.keys()))
service.set_error(error_msg)
raise ZoeStartExecutionFatalException("Service {} has wrong environment expression")
env_list.append((env_name, env_value))
return env_list
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The high-level interface that Zoe uses to talk to the configured container backend."""
import logging
from zoe_lib.config import get_conf
from zoe_lib.state import Execution, Service
from zoe_master.backends.base import BaseBackend
from zoe_master.backends.old_swarm.backend import OldSwarmBackend
log = logging.getLogger(__name__)
_backend_initialized = False
def _get_backend() -> BaseBackend:
backend_name = get_conf().backend
if backend_name == 'OldSwarm':
return OldSwarmBackend(get_conf())
else:
log.error('Unknown backend selected')
assert False