Commit f6b39c60 authored by Daniele Venzano's avatar Daniele Venzano

Merge branch 'master' into devel/zapp-shop, add documentation for installing Zoe

# Conflicts:
#	zoe_lib/config.py
parents 708667f2 c331579e
......@@ -4,26 +4,16 @@ directory=/home/ubuntu/zoe
autostart=true
autorestart=true
startretries=3
stderr_logfile=/var/log/zoe/master.err.log
stdout_logfile=/var/log/zoe/master.out.log
redirect_stderr=true
stdout_logfile=/var/log/zoe/master.log
user=ubuntu
[program:zoe-web]
command=/usr/bin/python3 /home/ubuntu/zoe/zoe-web.py
[program:zoe-api]
command=/usr/bin/python3 /home/ubuntu/zoe/zoe-api.py
directory=/home/ubuntu/zoe
autostart=true
autorestart=true
startretries=3
stderr_logfile=/var/log/zoe/web.err.log
stdout_logfile=/var/log/zoe/web.out.log
user=ubuntu
[program:zoe-observer]
command=/usr/bin/python3 /home/ubuntu/zoe/zoe-observer.py
directory=/home/ubuntu/zoe
autostart=true
autorestart=true
startretries=3
stderr_logfile=/var/log/zoe/observer.err.log
stdout_logfile=/var/log/zoe/observer.out.log
redirect_stderr=true
stdout_logfile=/var/log/zoe/web.log
user=ubuntu
......@@ -7,7 +7,7 @@ The main Zoe Components are:
* zoe master: the core component that performs application scheduling and talks to Swarm
* zoe api: the Zoe frontend, offering a web interface and a REST API
* zoe: command-line client
* command-line clients (zoe.py and zoe-admin.py)
The Zoe master is the core component of Zoe and communicates with the clients by using an internal ZeroMQ-based protocol. This protocol is designed to be robust, using the best practices from ZeroMQ documentation. A crash of the Api or of the Master process will not leave the other component inoperable, and when the faulted process restarts, work will restart where it was left.
......
......@@ -3,26 +3,31 @@
Zoe configuration
=================
Zoe can be configured by files, environment variables or commandline options. The configuration directives listed in this file can be specified by any of the three methods. Use the ``--help`` command-line option to have more details on the format of environment variables and precedence rules.
Zoe can be configured by files, environment variables or commandline options. Most of the configuration directives listed in this file can be specified by any of the three methods, with some exception outlined below. Use the ``--help`` command-line option to have more details on the format of environment variables and precedence rules.
The directive ``--write-config <filename>`` is also available: it will generate a configuration file with all options set to the default values.
Command-line options
--------------------
The Zoe config file have a simple format of ``<option name> = <value>``. Dash characters can be use for comments.
All Zoe processes use one single configuration file, called zoe.conf. It is searched in the current working directory and in ``/etc/zoe/``.
These options can be specified only on the command-line:
* ``--write-config <filename>`` : generate a configuration file with all options set to the default values
* ``--log-file <filename>`` : write the log output to the specified file instead of ``stderr``
zoe.conf
--------
The Zoe config file have a simple format of ``<option name> = <value>``. Dash characters can be use for comments.
All Zoe processes use one single configuration file, called zoe.conf. It is searched in the current working directory and in ``/etc/zoe/``.
Common options:
* ``debug = <true|false>`` : enable or disable debug log output
* ``api-listen-uri = tcp://*:4850`` : ZeroMQ server connection string, used for the master listening endpoint
* ``deployment-name = devel`` : name of this Zoe deployment. Can be used to have multiple Zoe deployments using the same back-end (devel and prod, for example)
Workspaces:
* ``workspace-deployment-path`` : path appended to the ``workspace-base-path`` to distinguish this deployment. If left unspecified it is equal to the deployment name
* ``workspace-base-path = /mnt/zoe-workspaces`` : Base directory where user workspaces will be created. This directory should reside on a shared filesystem visible by all Docker hosts.
* ``workspace-base-path = /mnt/zoe-workspaces`` : Base directory where user workspaces will be created. This directory should reside on a shared filesystem visible by all hosts where containers will be run.
Metrics:
......@@ -30,11 +35,11 @@ Metrics:
* ``influxdb-url = http://localhost:8086`` : URL of the InfluxDB service (ex. )
* ``influxdb-enable = False`` : Enable metric output toward influxDB
Service logs:
Service logs (see: :ref:`logging`):
* ``gelf-address = 7896``, help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:7896)', default='')
argparser.add_argument('--gelf-listener', type=int, help='Enable the internal GELF log listener on this port, set to 0 to disable', default='7896')
argparser.add_argument('--service-logs-base-path',
* ``gelf-address`` : Enable Docker GELF log output to a UDP listener (ex. udp://1.2.3.4:7896), works only for the Swarm back-end
* ``gelf-listener = 7896`` : Enable the internal GELF log listener on this port, set to 0 to disable
* ``service-logs-base-path = /var/lib/zoe/service-logs`` : Path where service logs coming from the GELF listener will be stored
PostgresQL database options:
......@@ -51,6 +56,14 @@ API options:
* ``master-url = tcp://127.0.0.1:4850`` : address of the Zoe Master ZeroMQ API
* ``cookie-secret = changeme``: secret used to encrypt cookies
Master options:
* ``api-listen-uri = tcp://*:4850`` : ZeroMQ server connection string, used for the master listening endpoint
Authentication:
* ``auth-type = text`` : Authentication type (text, ldap or ldapsasl)
* ``auth-file = zoepass.csv`` : Path to the CSV file containing user,pass,role lines for text authentication
* ``ldap-server-uri = ldap://localhost`` : LDAP server to use for user authentication
* ``ldap-base-dn = ou=something,dc=any,dc=local`` : LDAP base DN for users
* ``ldap-admin-gid = 5000`` : LDAP group ID for admins
......@@ -64,6 +77,10 @@ Scheduler options:
Default options for the scheduler enable the traditional Zoe scheduler that was already available in the previous releases.
ZApp shop:
* ``zapp-shop-path = /var/lib/zoe-apps`` : Path where ZApp folders are stored
Backend choice:
* ``backend = <Swarm|Kubernetes>`` : cluster back-end to use to run ZApps
......
......@@ -9,6 +9,12 @@ To better work together we have established some rules on how to contribute.
If you need ideas on features that are waiting to be implemented, you can check the `roadmap <https://github.com/DistributedSystemsGroup/zoe/wiki/RoadMap>`_.
Development repository
----------------------
Development happens at `Eurecom's GitLab repository <https://gitlab.eurecom.fr/zoe/main>`_. The GitHub repository is a read-only mirror.
The choice of GitLab over GitHub is due to the CI pipeline that we set-up to test Zoe.
Bug reports and feature requests
--------------------------------
......
This diff is collapsed.
......@@ -62,22 +62,23 @@ def load_configuration(test_conf=None):
argparser.add_argument('--influxdb-dbname', help='Name of the InfluxDB database to use for storing metrics', default='zoe')
argparser.add_argument('--influxdb-url', help='URL of the InfluxDB service (ex. http://localhost:8086)', default='http://localhost:8086')
argparser.add_argument('--influxdb-enable', action="store_true", help='Enable metric output toward influxDB')
argparser.add_argument('--workspace-base-path', help='Path where user workspaces will be created by Zoe. Must be visible at this path on all Swarm hosts.', default='/mnt/zoe-workspaces')
argparser.add_argument('--workspace-base-path', help='Base directory where user workspaces will be created. Must be visible at this path on all hosts.', default='/mnt/zoe-workspaces')
argparser.add_argument('--workspace-deployment-path', help='Path appended to the workspace path to distinguish this deployment. If unspecified is equal to the deployment name.', default='--default--')
argparser.add_argument('--overlay-network-name', help='Name of the Swarm overlay network Zoe should use', default='zoe')
# Service logs
argparser.add_argument('--gelf-address', help='Enable Docker GELF log output to this destination (ex. udp://1.2.3.4:7896)', default='')
argparser.add_argument('--gelf-listener', type=int, help='Enable the internal GELF log listener on this port, set to 0 to disable', default='7896')
argparser.add_argument('--service-logs-base-path', help='Path where service logs coming from the GELF listern will be stored', default='/var/lib/zoe/service-logs')
argparser.add_argument('--service-logs-base-path', help='Path where service logs coming from the GELF listener will be stored', default='/var/lib/zoe/service-logs')
# API options
argparser.add_argument('--listen-address', type=str, help='Address to listen to for incoming connections', default="0.0.0.0")
argparser.add_argument('--listen-port', type=int, help='Port to listen to for incoming connections', default=5001)
argparser.add_argument('--master-url', help='URL of the Zoe master process', default='tcp://127.0.0.1:4850')
argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
# API auth options
argparser.add_argument('--auth-type', help='Authentication type (text or ldap)', default='text')
argparser.add_argument('--auth-type', help='Authentication type (text, ldap or ldapsasl)', default='text')
argparser.add_argument('--auth-file', help='Path to the CSV file containing user,pass,role lines for text authentication', default='zoepass.csv')
......@@ -87,11 +88,10 @@ def load_configuration(test_conf=None):
argparser.add_argument('--ldap-user-gid', type=int, help='LDAP group ID for users', default=5001)
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
argparser.add_argument('--service-log-path', help='Save service logs in this directory', default='/tmp')
# Proxy options
argparser.add_argument('--proxy-path', help='Proxy base path', default='127.0.0.1')
# Scheduler
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', choices=['ZoeSimpleScheduler', 'ZoeElasticScheduler'], default='ZoeSimpleScheduler')
argparser.add_argument('--scheduler-policy', help='Scheduler policy to use for scheduling ZApps', choices=['FIFO', 'SIZE'], default='FIFO')
......@@ -110,7 +110,6 @@ def load_configuration(test_conf=None):
# other options
argparser.add_argument('--zapp-shop-path', help='Path where ZApp folders are stored', default='/var/lib/zoe-apps')
argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
argparser.add_argument('--log-file', help='output logs to a file', default='stderr')
argparser.add_argument('--aml-ttl', help='TimeToLive in hours for AML executions', type=int, default=4)
......
......@@ -304,7 +304,7 @@ class ArgumentParser(argparse.ArgumentParser):
argparse.ArgumentParser.__init__(self, **kwargs_for_super)
# parse the additionial args
# parse the additional args
if config_file_parser_class is None:
self._config_file_parser = DefaultConfigFileParser()
else:
......@@ -687,7 +687,7 @@ class ArgumentParser(argparse.ArgumentParser):
def format_values(self):
"""Returns a string with all args and settings and where they came from
(eg. commandline, config file, enviroment variable or default)
(eg. commandline, config file, environment variable or default)
"""
source_key_to_display_value_map = {
_COMMAND_LINE_SOURCE_KEY: "Command Line Args: ",
......@@ -730,8 +730,7 @@ class ArgumentParser(argparse.ArgumentParser):
config_path_actions = [a for a in
self._actions if getattr(a, "is_config_file_arg", False)]
if config_settable_args and (default_config_files or
config_path_actions):
if config_settable_args and (default_config_files or config_path_actions):
self._add_config_file_help = False # prevent duplication
added_config_file_help = True
......
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Container log storage"""
import logging
import os
import shutil
from zoe_lib.state.sql_manager import Execution
from zoe_lib.config import get_conf
from zoe_master.backends.swarm.api_client import SwarmClient
log = logging.getLogger(__name__)
def _path_from_execution(execution: Execution):
return os.path.join(get_conf().service_log_path, get_conf().deployment_name, str(execution.id))
def _init(execution: Execution):
if get_conf().service_log_path == '':
return None
base_path = _path_from_execution(execution)
try:
os.makedirs(base_path, exist_ok=True)
except (OSError, PermissionError):
log.exception('Error creating the directory at path: {}'.format(base_path))
return None
return base_path
def _shutdown():
pass
def save(execution: Execution):
"""Save the logs of the service specified as argument"""
path = _init(execution)
if path is None:
return
for service in execution.services:
fname = service.name + '.txt'
fpath = os.path.join(path, fname)
swarm = SwarmClient()
log_gen = swarm.logs(service.docker_id, stream=True, follow=False)
if log_gen is None:
_shutdown()
return
try:
with open(fpath, 'wb') as out_fp:
for line in log_gen:
out_fp.write(line)
except FileNotFoundError:
log.error("Could not create file {}".format(fpath))
_shutdown()
def delete(execution: Execution):
"""Delete the logs for a service"""
path = _init(execution)
if path is None:
return
shutil.rmtree(path, ignore_errors=True)
_shutdown()
......@@ -16,9 +16,11 @@
"""Layer in front of the scheduler to perform request pre-processing."""
import logging
import os
import shutil
import zoe_lib.exec_logs as exec_logs
from zoe_lib.state import Execution, SQLManager
from zoe_lib.config import get_conf
from zoe_master.scheduler import ZoeBaseScheduler
log = logging.getLogger(__name__)
......@@ -85,4 +87,8 @@ def restart_resubmit_scheduler(state: SQLManager, scheduler: ZoeBaseScheduler):
def execution_delete(execution: Execution):
"""Remove an execution, must only be called if the execution is NOT running."""
assert not execution.is_active
exec_logs.delete(execution)
path = os.path.join(get_conf().service_logs_base_path, get_conf().deployment_name, str(execution.id))
if path is None:
return
shutil.rmtree(path, ignore_errors=True)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment