Commit 045cc3cc authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Try not to fail completely in case of unexpected exceptions

parent 7a56de76
Pipeline #4696 failed with stages
in 37 seconds
......@@ -146,7 +146,7 @@ class DockerEngineBackend(zoe_master.backends.base.BaseBackend):
stats[cont['id']]['mem_usage'] = aux['memory_stats']['usage']
stats[cont['id']]['cpu_usage'] = self._get_core_usage(aux)
stats[cont['id']]['mem_limit'] = cont['memory_soft_limit']
except ZoeException:
except (ZoeException, KeyError):
continue
node_stats.memory_in_use = sum([stat['mem_usage'] for stat in stats.values()])
node_stats.cores_in_use = sum([stat['cpu_usage'] for stat in stats.values()])
......
......@@ -136,21 +136,38 @@ def start_all(execution: Execution) -> str:
"""
log.debug('starting all services for execution {}'.format(execution.id))
execution.set_starting()
return service_list_to_containers(execution, execution.services)
try:
ret = service_list_to_containers(execution, execution.services)
except BaseException:
log.exception('Unexpected exception while starting execution {}'.format(execution.id))
return 'requeue'
else:
return ret
def start_essential(execution: Execution, placement) -> str:
"""Start the essential services for this execution"""
log.debug('starting essential services for execution {}'.format(execution.id))
execution.set_starting()
return service_list_to_containers(execution, execution.essential_services, placement)
try:
ret = service_list_to_containers(execution, execution.essential_services, placement)
except BaseException:
log.exception('Unexpected exception while starting execution {}'.format(execution.id))
return 'requeue'
else:
return ret
def start_elastic(execution: Execution, placement) -> str:
"""Start the runnable elastic services"""
elastic_to_start = [s for s in execution.elastic_services if s.status == Service.RUNNABLE_STATUS]
return service_list_to_containers(execution, elastic_to_start, placement)
try:
ret = service_list_to_containers(execution, elastic_to_start, placement)
except BaseException:
log.exception('Unexpected exception while starting execution {}'.format(execution.id))
return 'requeue'
else:
return ret
def terminate_service(service: Service) -> None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment