Simplify self-termination state transitions

zoe_lib/state/service.py

@@ -233,9 +233,13 @@ class Service:
         if self.is_dead():
             for port in self.ports:
                 port.reset()
-            self.backend_id = None
             self.ip_address = None
-            self.sql_manager.service_update(self.id, backend_status=new_status, ip_address=None, backend_id=None)
+            self.sql_manager.service_update(self.id, backend_status=new_status, ip_address=None)
+            if new_status == self.BACKEND_DESTROY_STATUS:
+                self.backend_id = None
+                self.sql_manager.service_update(self.id, backend_status=new_status, backend_id=None, ip_address=None)
+            else:
+                self.sql_manager.service_update(self.id, backend_status=new_status, ip_address=None)
         else:
             self.sql_manager.service_update(self.id, backend_status=new_status)
 

zoe_master/backends/docker/backend.py

@@ -79,7 +79,10 @@ class DockerEngineBackend(zoe_master.backends.base.BaseBackend):
         """Terminate and delete a container."""
         conf = self._get_config(service.backend_host)
         engine = DockerClient(conf)
-        engine.terminate_container(service.backend_id, delete=True)
+        if service.backend_id is not None:
+            engine.terminate_container(service.backend_id, delete=True)
+        else:
+            log.error('Cannot terminate service {}, since it has not backend ID'.format(service.name))
         service.set_backend_status(service.BACKEND_DESTROY_STATUS)
 
     def platform_state(self) -> ClusterStats:

zoe_master/backends/docker/threads.py

@@ -66,28 +66,17 @@ class DockerStateSynchronizer(threading.Thread):
                 log.info('Node {} is now online'.format(host_config.name))
             node_status = 'online'
 
-            service_list = self.state.service_list(backend_host=host_config.name, not_status=Service.INACTIVE_STATUS)
             try:
                 container_list = my_engine.list(only_label={'zoe_deployment_name': get_conf().deployment_name})
             except ZoeException:
                 continue
-            containers = {}
+
             for cont in container_list:
-                containers[cont['id']] = cont
-
-            services = {}
-            for service in service_list:
-                services[service.backend_id] = service
-
-            for service in service_list:
-                assert isinstance(service, Service)
-                if service.backend_id in containers:
-                    self._update_service_status(service, containers[service.backend_id])
-                else:
-                    if service.status == service.CREATED_STATUS or service.backend_status == service.BACKEND_DESTROY_STATUS:
-                        continue
-                    else:
-                        service.set_backend_status(service.BACKEND_DESTROY_STATUS)
+                service = self.state.service_list(only_one=True, backend_host=host_config.name, backend_id=cont['id'])
+                if service is None:
+                    log.warning('Container {} on host {} has no corresponding service'.format(cont['name'], host_config.name))
+                    continue
+                self._update_service_status(service, cont)
 
             time.sleep(CHECK_INTERVAL)
 

zoe_master/backends/interface.py

@@ -155,7 +155,6 @@ def start_elastic(execution: Execution, placement) -> str:
 
 def terminate_execution(execution: Execution) -> None:
     """Terminate an execution."""
-    execution.set_cleaning_up()
     backend = _get_backend()
     for service in execution.services:  # type: Service
         if service.status != Service.INACTIVE_STATUS:
@@ -170,6 +169,12 @@ def terminate_execution(execution: Execution) -> None:
                 service.set_inactive()
             else:
                 log.error('BUG: don\'t know how to terminate a service in status {}'.format(service.status))
+        elif not service.is_dead():
+            log.warning('Service {} is inactive for Zoe, but running for the back-end, terminating and resetting state'.format(service.name))
+            service.set_terminating()
+            backend.terminate_service(service)
+            service.set_inactive()
+            log.debug('Service {} terminated'.format(service.name))
 
     execution.set_terminated()