Remove environment substitution and refactor Zoe-provided volumes

docs/zapps/zapp_packaging.rst

+.. _zapp_packaging:
+
+A new ZApp format
+=================
+
+* Remove the substitution language
+
+All information from Zoe is passed in predefined environment variables:
+
+* ZOE_EXECUTION_NAME
+* ZOE_EXECUTION_ID
+* ZOE_SERVICE_GROUP
+* ZOE_SERVICE_NAME
+* ZOE_SERVICE_ID
+* ZOE_OWNER
+* ZOE_DEPLOYMENT_NAME
+* ZOE_MY_DNS_NAME
+
+A number of volumes will be mounted on all containers:
+
+* User workspace, same for all containers of a certain user
+* Logs, different for each container
+
+A Zoe script will be the entrypoint for all ZApp containers. If a command is specified for a container, it must be a user-specified script that will be run by the Zoe script.
+
+
+Packaging
+---------
+
+ZApps will be distributed as self-contained set of files with this structure:
+
+* app.json : ZApp description (the current one with a few changes)
+* icon.png : an icon to be shown on graphical interfaces
+* metadata.json : metadata for the app and options that can be set to modify the app behaviour
+* docker/
+  * docker/image1/Dockerfile : the dockerfile needed to build the image for service 1
+  * docker/image1/* : all the other files required to build the image for service 1
+  * ...
+  * docker/imageN/Dockerfile : the dockerfile needed to build the image for service N
+  * docker/imageN/* : all the other files required to build the image for service N
+
+ZApps can be moved as tar/zip archives, downloaded via a website or cloned from github.

zoe_master/backends/base.py

@@ -34,7 +34,7 @@ class BaseBackend:
         """Performs a clean shutdown of the resources used by Swarm backend. Any threads that where started in the init() method should be terminated here. This method will be called when Zoe shuts down."""
         raise NotImplementedError
 
-    def spawn_service(self, execution: Execution, service: Service, env_subst_dict: Dict):
+    def spawn_service(self, execution: Execution, service: Service):
         """Create a container for a service."""
         raise NotImplementedError
 

zoe_master/backends/common.py

@@ -15,23 +15,38 @@
 
 """The high-level interface that Zoe uses to talk to the configured container backend."""
 
-from typing import Dict
+from zoe_lib.state import Service, Execution
+from zoe_lib.config import get_conf
+from zoe_master.workspace.filesystem import ZoeFSWorkspace
 
-from zoe_lib.state import Service
-from zoe_master.exceptions import ZoeStartExecutionFatalException
 
+def gen_environment(service: Service, execution: Execution):
+    """Return the list of environment variables that needs to be added to all containers."""
+    fswk = ZoeFSWorkspace()
+    env_list = [
+        ('ZOE_EXECUTION_ID', execution.id),
+        ('ZOE_EXECUTION_NAME', execution.name),
+        ('ZOE_SERVICE_GROUP', service.service_group),
+        ('ZOE_SERVICE_NAME', service.name),
+        ('ZOE_SERVICE_ID', service.id),
+        ('ZOE_OWNER', execution.user_id),
+        ('ZOE_DEPLOYMENT_NAME', get_conf().deployment_name),
+        ('ZOE_MY_DNS_NAME', service.dns_name),
+        ('ZOE_WORKSPACE', fswk.get_mountpoint())
+    ]
+    service_list = []
+    for tmp_service in execution.services:
+        service_list.append(tmp_service.dns_name)
+    env_list.append(('ZOE_EXECUTION_SERVICE_LIST', ','.join(service_list)))
 
-def gen_environment(service: Service, env_subst_dict: Dict):
-    """ Generate a dictionary containing the current cluster status (before the new container is spawned)
-
-    This information is used to substitute template strings in the environment variables."""
-    env_list = []
-    for env_name, env_value in service.environment:
-        try:
-            env_value = env_value.format(**env_subst_dict)
-        except KeyError:
-            error_msg = "Unknown variable in environment expression '{}', known variables are: {}".format(env_value, list(env_subst_dict.keys()))
-            service.set_error(error_msg)
-            raise ZoeStartExecutionFatalException("Service {} has wrong environment expression")
-        env_list.append((env_name, env_value))
     return env_list
+
+
+def gen_volumes(service: Service, execution: Execution):
+    """Return the list of default volumes to be added to all containers."""
+    vol_list = []
+    fswk = ZoeFSWorkspace()
+    if fswk.can_be_attached():
+        vol_list.append(fswk.get(execution.user_id))
+
+    return vol_list

zoe_master/backends/interface.py

@@ -59,21 +59,10 @@ def service_list_to_containers(execution: Execution, service_list: List[Service]
 
     ordered_service_list = sorted(service_list, key=lambda x: x.startup_order)
 
-    env_subst_dict = {
-        'execution_id': execution.id,
-        "execution_name": execution.name,
-        'user_name': execution.user_id,
-        'deployment_name': get_conf().deployment_name,
-    }
-
-    for service in ordered_service_list:
-        env_subst_dict['dns_name#' + service.name] = service.dns_name
-
     for service in ordered_service_list:
-        env_subst_dict['dns_name#self'] = service.dns_name
         service.set_starting()
         try:
-            backend.spawn_service(execution, service, env_subst_dict)
+            backend.spawn_service(execution, service)
         except ZoeStartExecutionRetryException as ex:
             log.warning('Temporary failure starting service {} of execution {}: {}'.format(service.id, execution.id, ex.message))
             execution.set_error_message(ex.message)

zoe_master/backends/old_swarm/backend.py

@@ -16,14 +16,12 @@
 """Zoe backend implementation for old-style stand-alone Docker Swarm."""
 
 import logging
-from typing import Dict
 
 from zoe_lib.config import get_conf
 from zoe_lib.exceptions import ZoeLibException, ZoeNotEnoughResourcesException
 from zoe_lib.state import Execution, Service
 from zoe_master.backends.old_swarm.api_client import DockerContainerOptions, SwarmClient
 from zoe_master.exceptions import ZoeStartExecutionRetryException, ZoeStartExecutionFatalException, ZoeException
-from zoe_master.workspace.filesystem import ZoeFSWorkspace
 import zoe_master.backends.common
 import zoe_master.backends.base
 from zoe_master.backends.old_swarm.threads import SwarmMonitor, SwarmStateSynchronizer
@@ -55,7 +53,7 @@ class OldSwarmBackend(zoe_master.backends.base.BaseBackend):
         _monitor.quit()
         _checker.quit()
 
-    def spawn_service(self, execution: Execution, service: Service, env_subst_dict: Dict):
+    def spawn_service(self, execution: Execution, service: Service):
         """Spawn a service, translating a Zoe Service into a Docker container."""
         copts = DockerContainerOptions()
         copts.gelf_log_address = get_conf().gelf_address
@@ -84,7 +82,7 @@ class OldSwarmBackend(zoe_master.backends.base.BaseBackend):
         # copts.restart = not service.is_monitor  # Monitor containers should not restart
         copts.restart = False
 
-        env_vars = zoe_master.backends.common.gen_environment(service, env_subst_dict)
+        env_vars = zoe_master.backends.common.gen_environment(service, execution)
         for name, value in env_vars:
             copts.add_env_variable(name, value)
 
@@ -92,7 +90,8 @@ class OldSwarmBackend(zoe_master.backends.base.BaseBackend):
             if port.expose:
                 copts.ports.append(port.port_number)
 
-        for volume in service.volumes:
+        zoe_volumes = zoe_master.backends.common.gen_volumes(service, execution)
+        for volume in service.volumes + zoe_volumes:
             if volume.type == "host_directory":
                 copts.add_volume_bind(volume.path, volume.mount_point, volume.readonly)
             else:
@@ -102,11 +101,6 @@ class OldSwarmBackend(zoe_master.backends.base.BaseBackend):
         #     for constraint in service.description['constraints']:
         #         copts.add_constraint(constraint)
 
-        fswk = ZoeFSWorkspace()
-        if fswk.can_be_attached():
-            copts.add_volume_bind(fswk.get_path(execution.user_id), fswk.get_mountpoint(), False)
-            copts.add_env_variable('ZOE_WORKSPACE', fswk.get_mountpoint())
-
         # The same dictionary is used for templates in the command
         copts.set_command(service.command.format(**env_subst_dict))
 

zoe_master/backends/old_swarm_new_api/backend.py

@@ -16,14 +16,12 @@
 """Zoe backend implementation for old-style stand-alone Docker Swarm."""
 
 import logging
-from typing import Dict
 
 from zoe_lib.config import get_conf
 from zoe_lib.exceptions import ZoeLibException, ZoeNotEnoughResourcesException
 from zoe_lib.state import Execution, Service
 from zoe_master.backends.old_swarm.api_client import DockerContainerOptions, SwarmClient
 from zoe_master.exceptions import ZoeStartExecutionRetryException, ZoeStartExecutionFatalException, ZoeException
-from zoe_master.workspace.filesystem import ZoeFSWorkspace
 import zoe_master.backends.common
 import zoe_master.backends.base
 from zoe_master.backends.old_swarm.threads import SwarmMonitor, SwarmStateSynchronizer
@@ -55,7 +53,7 @@ class OldSwarmNewAPIBackend(zoe_master.backends.base.BaseBackend):
         _monitor.quit()
         _checker.quit()
 
-    def spawn_service(self, execution: Execution, service: Service, env_subst_dict: Dict):
+    def spawn_service(self, execution: Execution, service: Service):
         """Spawn a service, translating a Zoe Service into a Docker container."""
         copts = DockerContainerOptions()
         copts.gelf_log_address = get_conf().gelf_address
@@ -84,7 +82,7 @@ class OldSwarmNewAPIBackend(zoe_master.backends.base.BaseBackend):
         # copts.restart = not service.is_monitor  # Monitor containers should not restart
         copts.restart = False
 
-        env_vars = zoe_master.backends.common.gen_environment(service, env_subst_dict)
+        env_vars = zoe_master.backends.common.gen_environment(service, execution)
         for name, value in env_vars:
             copts.add_env_variable(name, value)
 
@@ -92,7 +90,8 @@ class OldSwarmNewAPIBackend(zoe_master.backends.base.BaseBackend):
             if port.expose:
                 copts.ports.append(port.number)
 
-        for volume in service.volumes:
+        zoe_volumes = zoe_master.backends.common.gen_volumes(service, execution)
+        for volume in service.volumes + zoe_volumes:
             if volume.type == "host_directory":
                 copts.add_volume_bind(volume.path, volume.mount_point, volume.readonly)
             else:
@@ -102,11 +101,6 @@ class OldSwarmNewAPIBackend(zoe_master.backends.base.BaseBackend):
         #     for constraint in service.description['constraints']:
         #         copts.add_constraint(constraint)
 
-        fswk = ZoeFSWorkspace()
-        if fswk.can_be_attached():
-            copts.add_volume_bind(fswk.get_path(execution.user_id), fswk.get_mountpoint(), False)
-            copts.add_env_variable('ZOE_WORKSPACE', fswk.get_mountpoint())
-
         # The same dictionary is used for templates in the command
         copts.set_command(service.command.format(**env_subst_dict))
 

zoe_master/workspace/filesystem.py

@@ -19,6 +19,7 @@ import logging
 import os.path
 
 import zoe_lib.config as config
+from zoe_lib.state.service import VolumeDescription
 import zoe_master.workspace.base
 
 log = logging.getLogger(__name__)
@@ -46,3 +47,7 @@ class ZoeFSWorkspace(zoe_master.workspace.base.ZoeWorkspaceBase):
     def get_mountpoint(cls):
         """Get the volume mount point."""
         return '/mnt/workspace'
+
+    def get(self, user_id):
+        """Return a VolumeDescription."""
+        return VolumeDescription((self.get_path(user_id), self.get_mountpoint(), False))