Commit e1a9d264 authored by Daniele Venzano's avatar Daniele Venzano

Integrate the elastic scheduler

parent abb334e6
......@@ -84,7 +84,8 @@ def load_configuration(test_conf=None):
argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', default='ZoeSimpleScheduler')
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', choices=['ZoeSimpleScheduler', 'ZoeElasticScheduler'], default='ZoeSimpleScheduler')
argparser.add_argument('--scheduler-policy', help='Scheduler policy to use for scheduling ZApps', choices=['FIFO', 'SIZE'], default='FIFO')
argparser.add_argument('--backend', choices=['OldSwarm'], default='OldSwarm')
......
......@@ -17,6 +17,7 @@
import datetime
import logging
import threading
log = logging.getLogger(__name__)
......@@ -66,6 +67,8 @@ class Execution:
self.priority = self.description['priority']
self.termination_lock = threading.Lock()
def serialize(self):
"""Generates a dictionary that can be serialized in JSON."""
return {
......@@ -122,6 +125,7 @@ class Execution:
self.error_message = message
self.sql_manager.execution_update(self.id, error_message=self.error_message)
@property
def is_active(self):
"""
Returns True if the execution is in the scheduler
......@@ -129,6 +133,7 @@ class Execution:
"""
return self._status == self.SCHEDULED_STATUS or self._status == self.RUNNING_STATUS or self._status == self.STARTING_STATUS or self._status == self.CLEANING_UP_STATUS
@property
def is_running(self):
"""Returns True is the execution has at least the essential services running."""
return self._status == self.RUNNING_STATUS
......@@ -143,6 +148,16 @@ class Execution:
"""Getter for this execution service list."""
return self.sql_manager.service_list(execution_id=self.id)
@property
def essential_services(self):
"""Getter for this execution essential service list."""
return self.sql_manager.service_list(execution_id=self.id, essential=True)
@property
def elastic_services(self):
"""Getter for this execution elastic service list."""
return self.sql_manager.service_list(execution_id=self.id, essential=False)
@property
def essential_services_running(self) -> bool:
"""Returns True if all essential services of this execution have started."""
......@@ -177,3 +192,6 @@ class Execution:
def total_reservations(self):
"""Return the union/sum of resources reserved by all services of this execution."""
return sum([s.resource_reservation for s in self.services])
def __repr__(self):
return str(self.id)
......@@ -64,6 +64,7 @@ class Service:
ACTIVE_STATUS = "active"
STARTING_STATUS = "starting"
ERROR_STATUS = "error"
RUNNABLE_STATUS = "runnable"
BACKEND_UNDEFINED_STATUS = 'undefined'
BACKEND_CREATE_STATUS = 'created'
......
......@@ -18,6 +18,7 @@
from typing import Dict
from zoe_lib.state import Execution, Service
from zoe_master.stats import ClusterStats
class BaseBackend:
......@@ -38,3 +39,7 @@ class BaseBackend:
def terminate_service(self, service: Service) -> None:
raise NotImplementedError
def platform_state(self) -> ClusterStats:
"""Get the platform state."""
raise NotImplementedError
......@@ -16,17 +16,17 @@
"""The high-level interface that Zoe uses to talk to the configured container backend."""
import logging
from typing import List
from zoe_lib.config import get_conf
from zoe_lib.state import Execution, Service
from zoe_master.backends.base import BaseBackend
from zoe_master.backends.old_swarm.backend import OldSwarmBackend
from zoe_master.exceptions import ZoeStartExecutionFatalException, ZoeStartExecutionRetryException
log = logging.getLogger(__name__)
_backend_initialized = False
def _get_backend() -> BaseBackend:
backend_name = get_conf().backend
......@@ -39,27 +39,21 @@ def _get_backend() -> BaseBackend:
def initialize_backend(state):
"""Initializes the configured backend."""
assert not _backend_initialized
backend = _get_backend()
backend.init(state)
def shutdown_backend():
"""Shuts down the configured backend."""
assert _backend_initialized
backend = _get_backend()
backend.shutdown()
def execution_to_containers(execution: Execution) -> None:
"""Translate an execution object into containers.
If an error occurs some containers may have been created and needs to be cleaned-up.
In case of error exceptions are raised.
"""
def service_list_to_containers(execution: Execution, service_list: List[Service]) -> str:
"""Given a subset of services from an execution, tries to start them, return one of 'ok', 'requeue' for temporary failures and 'fatal' for fatal failures."""
backend = _get_backend()
ordered_service_list = sorted(execution.services, key=lambda x: x.startup_order)
ordered_service_list = sorted(service_list, key=lambda x: x.startup_order)
env_subst_dict = {
'execution_id': execution.id,
......@@ -74,7 +68,54 @@ def execution_to_containers(execution: Execution) -> None:
for service in ordered_service_list:
env_subst_dict['dns_name#self'] = service.dns_name
service.set_starting()
backend.spawn_service(execution, service, env_subst_dict)
try:
backend.spawn_service(execution, service, env_subst_dict)
except ZoeStartExecutionRetryException as ex:
log.warning('Temporary failure starting service {} of execution {}: {}'.format(service.id, execution.id, ex.message))
execution.set_error_message(ex.message)
terminate_execution(execution)
execution.set_scheduled()
return "requeue"
except ZoeStartExecutionFatalException as ex:
log.error('Fatal error trying to start service {} of execution {}: {}'.format(service.id, execution.id, ex.message))
execution.set_error_message(ex.message)
terminate_execution(execution)
execution.set_error()
return "fatal"
except Exception as ex:
log.error('Fatal error trying to start service {} of execution {}'.format(service.id, execution.id))
log.exception('BUG, this error should have been caught earlier')
execution.set_error_message(str(ex))
terminate_execution(execution)
execution.set_error()
return "fatal"
else:
execution.set_running()
return "ok"
def start_all(execution: Execution) -> str:
"""Translate an execution object into containers.
If an error occurs some containers may have been created and needs to be cleaned-up.
"""
log.debug('starting all services for execution {}'.format(execution.id))
execution.set_starting()
return service_list_to_containers(execution, execution.services)
def start_essential(execution) -> str:
"""Start the essential services for this execution"""
log.debug('starting essential services for execution {}'.format(execution.id))
execution.set_starting()
return service_list_to_containers(execution, execution.essential_services)
def start_elastic(execution) -> str:
"""Start the runnable elastic services"""
elastic_to_start = [s for s in execution.elastic_services if s.status == Service.RUNNABLE_STATUS]
return service_list_to_containers(execution, elastic_to_start)
def terminate_execution(execution: Execution) -> None:
......@@ -89,3 +130,9 @@ def terminate_execution(execution: Execution) -> None:
service.set_inactive()
log.debug('Service {} terminated'.format(service.name))
execution.set_terminated()
def get_platform_state():
"""Retrieves the state of the platform by querying the container backend. Platform state includes information on free/reserved resources for each node. This information is used for advanced scheduling."""
backend = _get_backend()
return backend.platform_state()
......@@ -27,6 +27,7 @@ from zoe_master.workspace.filesystem import ZoeFSWorkspace
import zoe_master.backends.common
import zoe_master.backends.base
from zoe_master.backends.old_swarm.threads import SwarmMonitor, SwarmStateSynchronizer
from zoe_master.stats import NodeStats, ClusterStats
log = logging.getLogger(__name__)
......@@ -124,3 +125,11 @@ class OldSwarmBackend(zoe_master.backends.base.BaseBackend):
def terminate_service(self, service: Service) -> None:
"""Terminate and delete a container."""
self.swarm.terminate_container(service.backend_id, delete=True)
def platform_state(self) -> ClusterStats:
"""Get the platform state."""
info = self.swarm.info()
for node in info.nodes: # type: NodeStats
node.memory_free = node.memory_total - node.memory_reserved
node.cores_free = node.cores_total - node.cores_reserved
return info
......@@ -54,7 +54,7 @@ def main():
state = SQLManager(args)
log.info("Initializing scheduler")
scheduler = getattr(zoe_master.scheduler, config.get_conf().scheduler_class)(state)
scheduler = getattr(zoe_master.scheduler, config.get_conf().scheduler_class)(state, config.get_conf().scheduler_policy)
zoe_master.backends.interface.initialize_backend(state)
......
......@@ -36,3 +36,8 @@ class ZoeStartExecutionRetryException(ZoeException):
class ZoeStartExecutionFatalException(ZoeException):
"""Execution emitted in case the Execution failed to start for a fatal error."""
pass
class UnsupportedSchedulerPolicyError(ZoeException):
"""The configuration asks for a combination of scheduler and policy that is unsupported."""
pass
......@@ -16,4 +16,5 @@
"""The Zoe schedulers"""
from .base_scheduler import ZoeBaseScheduler
from .scheduler import ZoeSimpleScheduler
from .simple_scheduler import ZoeSimpleScheduler
from .elastic_scheduler import ZoeElasticScheduler
This diff is collapsed.
......@@ -19,17 +19,19 @@ import logging
import threading
from zoe_lib.state import Execution
from zoe_master.backends.interface import execution_to_containers, terminate_execution
from zoe_master.exceptions import ZoeStartExecutionFatalException, ZoeStartExecutionRetryException
from zoe_master.backends.interface import start_all, terminate_execution
from zoe_master.scheduler.base_scheduler import ZoeBaseScheduler
from zoe_master.exceptions import UnsupportedSchedulerPolicyError
log = logging.getLogger(__name__)
class ZoeSimpleScheduler(ZoeBaseScheduler):
"""The Scheduler class."""
def __init__(self, state):
def __init__(self, state, policy):
super().__init__(state)
if policy != 'FIFO':
raise UnsupportedSchedulerPolicyError
self.fifo_queue = []
self.trigger_semaphore = threading.Semaphore(0)
self.async_threads = []
......@@ -103,26 +105,9 @@ class ZoeSimpleScheduler(ZoeBaseScheduler):
e.set_starting()
self.fifo_queue.pop(0) # remove the execution form the queue
try:
execution_to_containers(e)
except ZoeStartExecutionRetryException as ex:
log.warning('Temporary failure starting execution {}: {}'.format(e.id, ex.message))
e.set_error_message(ex.message)
terminate_execution(e)
e.set_scheduled()
ret = start_all(e)
if ret == 'requeue':
self.fifo_queue.append(e)
except ZoeStartExecutionFatalException as ex:
log.error('Fatal error trying to start execution {}: {}'.format(e.id, ex.message))
e.set_error_message(ex.message)
terminate_execution(e)
e.set_error()
except Exception as ex:
log.exception('BUG, this error should have been caught earlier')
e.set_error_message(str(ex))
terminate_execution(e)
e.set_error()
else:
e.set_running()
def quit(self):
"""Stop the scheduler thread."""
......
"""Classes to hold the system state and simulated container/service placements"""
from zoe_lib.state.sql_manager import Execution, Service
from zoe_master.stats import ClusterStats, NodeStats
class SimulatedNode:
"""A simulated node where containers can be run"""
def __init__(self, real_node: NodeStats):
self.real_reservations = {
"memory": real_node.memory_reserved
}
self.real_free_resources = {
"memory": real_node.memory_free
}
self.real_active_containers = real_node.container_count
self.services = []
self.name = real_node.name
def service_fits(self, service: Service) -> bool:
"""Checks whether a service can fit in this node"""
if service.resource_reservation.memory < self.node_free_memory():
return True
else:
return False
def service_add(self, service):
"""Add a service in this node."""
if self.service_fits(service):
self.services.append(service)
return True
else:
return False
def service_remove(self, service):
"""Add a service in this node."""
try:
self.services.remove(service)
except ValueError:
return False
else:
return True
@property
def container_count(self):
"""Return the number of containers on this node"""
return self.real_active_containers + len(self.services)
def node_free_memory(self):
"""Return the amount of free memory for this node"""
simulated_reservation = 0
for service in self.services: # type: Service
simulated_reservation += service.resource_reservation.memory
assert (self.real_free_resources['memory'] - simulated_reservation) >= 0
return self.real_free_resources['memory'] - simulated_reservation
def __repr__(self):
# services = ','.join([str(s.id) for s in self.services])
s = 'SN {} | f {}'.format(self.name, self.node_free_memory())
return s
class SimulatedPlatform:
"""A simulated cluster, composed by simulated nodes"""
def __init__(self, plastform_status: ClusterStats):
self.nodes = {}
for node in plastform_status.nodes:
self.nodes[node.name] = SimulatedNode(node)
def allocate_essential(self, execution: Execution) -> bool:
"""Try to find an allocation for essential services"""
for service in execution.essential_services:
candidate_nodes = []
for node_name, node in self.nodes.items():
if node.service_fits(service):
candidate_nodes.append(node)
if len(candidate_nodes) == 0: # this service does not fit anywhere
self.deallocate_essential(execution)
return False
candidate_nodes.sort(key=lambda n: n.container_count) # smallest first
candidate_nodes[0].service_add(service)
return True
def deallocate_essential(self, execution: Execution):
"""Remove all essential services from the simulated cluster"""
for service in execution.essential_services:
for node_name, node in self.nodes.items():
if node.service_remove(service):
break
def allocate_elastic(self, execution: Execution) -> bool:
"""Try to find an allocation for elastic services"""
at_least_one_allocated = False
for service in execution.elastic_services:
if service.status == service.ACTIVE_STATUS:
continue
candidate_nodes = []
for node_name, node in self.nodes.items():
if node.service_fits(service):
candidate_nodes.append(node)
if len(candidate_nodes) == 0: # this service does not fit anywhere
continue
candidate_nodes.sort(key=lambda n: n.container_count) # smallest first
candidate_nodes[0].service_add(service)
service.set_runnable()
at_least_one_allocated = True
return at_least_one_allocated
def deallocate_elastic(self, execution: Execution):
"""Remove all elastic services from the simulated cluster"""
for service in execution.elastic_services:
for node_name, node in self.nodes.items():
if node.service_remove(service):
service.set_inactive()
break
def aggregated_free_memory(self):
"""Return the amount of free memory across all nodes"""
total = 0
for n_id, n in self.nodes.items():
total += n.node_free_memory()
return total
def get_service_allocation(self):
"""Return a map of service IDs to nodes where they have been allocated."""
placements = {}
for node_id, node in self.nodes.items():
for service in node.services:
placements[service.id] = node_id
return placements
def __repr__(self):
s = ''
for node_name, node in self.nodes.items():
s += str(node) + " # "
return s
......@@ -35,8 +35,10 @@ class NodeStats(Stats):
self.container_count = 0
self.cores_total = 0
self.cores_reserved = 0
self.cores_free = 0
self.memory_total = 0
self.memory_reserved = 0
self.memory_free = 0
self.labels = {}
self.status = None
self.error = ''
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment