Clean-ups and update documentation for configuration

docs/config_file.rst

@@ -3,26 +3,31 @@
 Zoe configuration
 =================
 
-Zoe can be configured by files, environment variables or commandline options. The configuration directives listed in this file can be specified by any of the three methods. Use the ``--help`` command-line option to have more details on the format of environment variables and precedence rules.
+Zoe can be configured by files, environment variables or commandline options. Most of the configuration directives listed in this file can be specified by any of the three methods, with some exception outlined below. Use the ``--help`` command-line option to have more details on the format of environment variables and precedence rules.
 
-The directive ``--write-config <filename>`` is also available: it will generate a configuration file with all options set to the default values.
+Command-line options
+--------------------
 
-The Zoe config file have a simple format of ``<option name> = <value>``. Dash characters can be use for comments.
-All Zoe processes use one single configuration file, called zoe.conf. It is searched in the current working directory and in ``/etc/zoe/``.
+These options can be specified only on the command-line:
+
+* ``--write-config <filename>`` : generate a configuration file with all options set to the default values
+* ``--log-file <filename>`` : write the log output to the specified file instead of ``stderr``
 
 zoe.conf
 --------
 
+The Zoe config file have a simple format of ``<option name> = <value>``. Dash characters can be use for comments.
+All Zoe processes use one single configuration file, called zoe.conf. It is searched in the current working directory and in ``/etc/zoe/``.
+
 Common options:
 
 * ``debug = <true|false>`` : enable or disable debug log output
-* ``api-listen-uri = tcp://*:4850`` : ZeroMQ server connection string, used for the master listening endpoint
 * ``deployment-name = devel`` : name of this Zoe deployment. Can be used to have multiple Zoe deployments using the same back-end (devel and prod, for example)
 
 Workspaces:
 
 * ``workspace-deployment-path`` : path appended to the ``workspace-base-path`` to distinguish this deployment. If left unspecified it is equal to the deployment name
-* ``workspace-base-path = /mnt/zoe-workspaces`` : Base directory where user workspaces will be created. This directory should reside on a shared filesystem visible by all Docker hosts.
+* ``workspace-base-path = /mnt/zoe-workspaces`` : Base directory where user workspaces will be created. This directory should reside on a shared filesystem visible by all hosts where containers will be run.
 
 Metrics:
 
@@ -30,11 +35,11 @@ Metrics:
 * ``influxdb-url = http://localhost:8086`` : URL of the InfluxDB service (ex. )
 * ``influxdb-enable = False`` : Enable metric output toward influxDB
 
-Service logs:
+Service logs (see: :ref:`logging`):
 
-* ``gelf-address = 7896``, help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:7896)', default='')
-        argparser.add_argument('--gelf-listener', type=int, help='Enable the internal GELF log listener on this port, set to 0 to disable', default='7896')
-        argparser.add_argument('--service-logs-base-path',
+* ``gelf-address`` : Enable Docker GELF log output to a UDP listener (ex. udp://1.2.3.4:7896), works only for the Swarm back-end
+* ``gelf-listener = 7896`` : Enable the internal GELF log listener on this port, set to 0 to disable
+* ``service-logs-base-path = /var/lib/zoe/service-logs`` : Path where service logs coming from the GELF listener will be stored
 
 PostgresQL database options:
 
@@ -51,6 +56,14 @@ API options:
 * ``master-url = tcp://127.0.0.1:4850`` : address of the Zoe Master ZeroMQ API
 * ``cookie-secret = changeme``: secret used to encrypt cookies
 
+Master options:
+
+* ``api-listen-uri = tcp://*:4850`` : ZeroMQ server connection string, used for the master listening endpoint
+
+Authentication:
+
+* ``auth-type = text`` : Authentication type (text or ldap)
+* ``auth-file = zoepass.csv`` : Path to the CSV file containing user,pass,role lines for text authentication
 * ``ldap-server-uri = ldap://localhost`` : LDAP server to use for user authentication
 * ``ldap-base-dn = ou=something,dc=any,dc=local`` : LDAP base DN for users
 * ``ldap-admin-gid = 5000`` : LDAP group ID for admins

zoe_lib/config.py

@@ -62,19 +62,20 @@ def load_configuration(test_conf=None):
         argparser.add_argument('--influxdb-dbname', help='Name of the InfluxDB database to use for storing metrics', default='zoe')
         argparser.add_argument('--influxdb-url', help='URL of the InfluxDB service (ex. http://localhost:8086)', default='http://localhost:8086')
         argparser.add_argument('--influxdb-enable', action="store_true", help='Enable metric output toward influxDB')
-        argparser.add_argument('--workspace-base-path', help='Path where user workspaces will be created by Zoe. Must be visible at this path on all Swarm hosts.', default='/mnt/zoe-workspaces')
+        argparser.add_argument('--workspace-base-path', help='Base directory where user workspaces will be created. Must be visible at this path on all hosts.', default='/mnt/zoe-workspaces')
         argparser.add_argument('--workspace-deployment-path', help='Path appended to the workspace path to distinguish this deployment. If unspecified is equal to the deployment name.', default='--default--')
         argparser.add_argument('--overlay-network-name', help='Name of the Swarm overlay network Zoe should use', default='zoe')
 
         # Service logs
         argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:7896)', default='')
         argparser.add_argument('--gelf-listener', type=int, help='Enable the internal GELF log listener on this port, set to 0 to disable', default='7896')
-        argparser.add_argument('--service-logs-base-path', help='Path where service logs coming from the GELF listern will be stored', default='/var/lib/zoe/service-logs')
+        argparser.add_argument('--service-logs-base-path', help='Path where service logs coming from the GELF listener will be stored', default='/var/lib/zoe/service-logs')
 
         # API options
         argparser.add_argument('--listen-address', type=str, help='Address to listen to for incoming connections', default="0.0.0.0")
         argparser.add_argument('--listen-port', type=int, help='Port to listen to for incoming connections', default=5001)
         argparser.add_argument('--master-url', help='URL of the Zoe master process', default='tcp://127.0.0.1:4850')
+        argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
 
         # API auth options
         argparser.add_argument('--auth-type', help='Authentication type (text or ldap)', default='text')
@@ -87,11 +88,10 @@ def load_configuration(test_conf=None):
         argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
         argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
 
-        argparser.add_argument('--service-log-path', help='Save service logs in this directory', default='/tmp')
-
         # Proxy options
         argparser.add_argument('--proxy-path', help='Proxy base path', default='127.0.0.1')
 
+        # Scheduler
         argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', choices=['ZoeSimpleScheduler', 'ZoeElasticScheduler'], default='ZoeSimpleScheduler')
         argparser.add_argument('--scheduler-policy', help='Scheduler policy to use for scheduling ZApps', choices=['FIFO', 'SIZE'], default='FIFO')
 
@@ -108,7 +108,6 @@ def load_configuration(test_conf=None):
         argparser.add_argument('--kube-config-file', help='Kubernetes configuration file', default='/opt/zoe/kube.conf')
         argparser.add_argument('--kube-namespace', help='The namespace that Zoe operates on', default='default')
 
-        argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
         argparser.add_argument('--log-file', help='output logs to a file', default='stderr')
 
         argparser.add_argument('--aml-ttl', help='TimeToLive in hours for AML executions', type=int, default=4)

zoe_lib/configargparse.py

@@ -304,7 +304,7 @@ class ArgumentParser(argparse.ArgumentParser):
 
         argparse.ArgumentParser.__init__(self, **kwargs_for_super)
 
-        # parse the additionial args
+        # parse the additional args
         if config_file_parser_class is None:
             self._config_file_parser = DefaultConfigFileParser()
         else:
@@ -687,7 +687,7 @@ class ArgumentParser(argparse.ArgumentParser):
 
     def format_values(self):
         """Returns a string with all args and settings and where they came from
-        (eg. commandline, config file, enviroment variable or default)
+        (eg. commandline, config file, environment variable or default)
         """
         source_key_to_display_value_map = {
             _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ",
@@ -730,8 +730,7 @@ class ArgumentParser(argparse.ArgumentParser):
             config_path_actions = [a for a in
                                    self._actions if getattr(a, "is_config_file_arg", False)]
 
-            if config_settable_args and (default_config_files or
-                                             config_path_actions):
+            if config_settable_args and (default_config_files or config_path_actions):
                 self._add_config_file_help = False  # prevent duplication
                 added_config_file_help = True
 

zoe_lib/exec_logs.py

-# Copyright (c) 2016, Daniele Venzano
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Container log storage"""
-
-import logging
-import os
-import shutil
-
-from zoe_lib.state.sql_manager import Execution
-from zoe_lib.config import get_conf
-from zoe_master.backends.swarm.api_client import SwarmClient
-
-log = logging.getLogger(__name__)
-
-
-def _path_from_execution(execution: Execution):
-    return os.path.join(get_conf().service_log_path, get_conf().deployment_name, str(execution.id))
-
-
-def _init(execution: Execution):
-    if get_conf().service_log_path == '':
-        return None
-    base_path = _path_from_execution(execution)
-    try:
-        os.makedirs(base_path, exist_ok=True)
-    except (OSError, PermissionError):
-        log.exception('Error creating the directory at path: {}'.format(base_path))
-        return None
-
-    return base_path
-
-
-def _shutdown():
-    pass
-
-
-def save(execution: Execution):
-    """Save the logs of the service specified as argument"""
-    path = _init(execution)
-    if path is None:
-        return
-
-    for service in execution.services:
-        fname = service.name + '.txt'
-        fpath = os.path.join(path, fname)
-
-        swarm = SwarmClient()
-        log_gen = swarm.logs(service.docker_id, stream=True, follow=False)
-        if log_gen is None:
-            _shutdown()
-            return
-        try:
-            with open(fpath, 'wb') as out_fp:
-                for line in log_gen:
-                    out_fp.write(line)
-        except FileNotFoundError:
-            log.error("Could not create file {}".format(fpath))
-
-    _shutdown()
-
-
-def delete(execution: Execution):
-    """Delete the logs for a service"""
-    path = _init(execution)
-    if path is None:
-        return
-
-    shutil.rmtree(path, ignore_errors=True)
-
-    _shutdown()

zoe_master/preprocessing.py

@@ -16,9 +16,11 @@
 """Layer in front of the scheduler to perform request pre-processing."""
 
 import logging
+import os
+import shutil
 
-import zoe_lib.exec_logs as exec_logs
 from zoe_lib.state import Execution, SQLManager
+from zoe_lib.config import get_conf
 from zoe_master.scheduler import ZoeBaseScheduler
 
 log = logging.getLogger(__name__)
@@ -85,4 +87,8 @@ def restart_resubmit_scheduler(state: SQLManager, scheduler: ZoeBaseScheduler):
 def execution_delete(execution: Execution):
     """Remove an execution, must only be called if the execution is NOT running."""
     assert not execution.is_active
-    exec_logs.delete(execution)
+    path = os.path.join(get_conf().service_logs_base_path, get_conf().deployment_name, str(execution.id))
+    if path is None:
+        return
+
+    shutil.rmtree(path, ignore_errors=True)