interface.py 9.27 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""The high-level interface that Zoe uses to talk to the configured container backend."""

import logging
19
import time
Daniele Venzano's avatar
Daniele Venzano committed
20
from typing import List
21
22

from zoe_lib.config import get_conf
23
from zoe_lib.state import Execution, Service, SQLManager  # pylint: disable=unused-import
24
25

from zoe_master.backends.base import BaseBackend
26
27
from zoe_master.backends.service_instance import ServiceInstance
from zoe_master.exceptions import ZoeStartExecutionFatalException, ZoeStartExecutionRetryException, ZoeException
28
from zoe_master.stats import ClusterStats, NodeStats  # pylint: disable=unused-import
29
30
31
32
33
34
35
36
37
38

try:
    from zoe_master.backends.swarm.backend import SwarmBackend
except ImportError as ex:
    SwarmBackend = None

try:
    from zoe_master.backends.kubernetes.backend import KubernetesBackend
except ImportError:
    KubernetesBackend = None
39

40
41
42
43
44
try:
    from zoe_master.backends.docker.backend import DockerEngineBackend
except ImportError:
    DockerEngineBackend = None

45
46
47
48
log = logging.getLogger(__name__)


def _get_backend() -> BaseBackend:
Daniele Venzano's avatar
Daniele Venzano committed
49
    """Return the right backend instance by reading the global configuration."""
50
    backend_name = get_conf().backend
Daniele Venzano's avatar
Daniele Venzano committed
51
    if backend_name == 'Kubernetes':
52
53
        if KubernetesBackend is None:
            raise ZoeException('The Kubernetes backend requires the pykube module')
hxquangnhat's avatar
hxquangnhat committed
54
        return KubernetesBackend(get_conf())
55
56
57
58
    elif backend_name == 'Swarm':
        if SwarmBackend is None:
            raise ZoeException('The Swarm backend requires docker python version >= 2.0.2')
        return SwarmBackend(get_conf())
59
60
61
62
    elif backend_name == 'DockerEngine':
        if DockerEngineBackend is None:
            raise ZoeException('The Docker Engine backend requires docker python version >= 2.0.2')
        return DockerEngineBackend(get_conf())
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    else:
        log.error('Unknown backend selected')
        assert False


def initialize_backend(state):
    """Initializes the configured backend."""
    backend = _get_backend()
    backend.init(state)


def shutdown_backend():
    """Shuts down the configured backend."""
    backend = _get_backend()
    backend.shutdown()


80
def service_list_to_containers(execution: Execution, service_list: List[Service], placement=None) -> str:
81
    """Given a subset of services from an execution, tries to start them, return one of 'ok', 'requeue' for temporary failures and 'fatal' for fatal failures."""
82
83
    backend = _get_backend()

84
    ordered_service_list = sorted(service_list, key=lambda x: x.startup_order)
85
86
87
88
89
90
91
92

    env_subst_dict = {
        'execution_id': execution.id,
        "execution_name": execution.name,
        'user_name': execution.user_id,
        'deployment_name': get_conf().deployment_name,
    }

93
    for service in execution.services:
94
95
96
97
        env_subst_dict['dns_name#' + service.name] = service.dns_name

    for service in ordered_service_list:
        env_subst_dict['dns_name#self'] = service.dns_name
98
99
        if placement is not None:
            service.assign_backend_host(placement[service.id])
100
        service.set_starting()
101
        instance = ServiceInstance(execution, service, env_subst_dict)
102
        try:
103
            backend_id, ip_address, ports = backend.spawn_service(instance)
104
105
        except ZoeStartExecutionRetryException as ex:
            log.warning('Temporary failure starting service {} of execution {}: {}'.format(service.id, execution.id, ex.message))
106
            service.set_error(ex.message)
107
108
109
110
111
112
            execution.set_error_message(ex.message)
            terminate_execution(execution)
            execution.set_scheduled()
            return "requeue"
        except ZoeStartExecutionFatalException as ex:
            log.error('Fatal error trying to start service {} of execution {}: {}'.format(service.id, execution.id, ex.message))
Daniele Venzano's avatar
Daniele Venzano committed
113
            service.set_error(ex.message)
114
115
116
117
118
119
120
121
122
123
124
125
            execution.set_error_message(ex.message)
            terminate_execution(execution)
            execution.set_error()
            return "fatal"
        except Exception as ex:
            log.error('Fatal error trying to start service {} of execution {}'.format(service.id, execution.id))
            log.exception('BUG, this error should have been caught earlier')
            execution.set_error_message(str(ex))
            terminate_execution(execution)
            execution.set_error()
            return "fatal"
        else:
126
            log.debug('Service {} started'.format(instance.name))
127
            service.set_active(backend_id, ip_address, ports)
hxquangnhat's avatar
hxquangnhat committed
128

qhoangxuan's avatar
qhoangxuan committed
129
    return "ok"
130
131
132
133
134
135
136
137
138
139
140
141


def start_all(execution: Execution) -> str:
    """Translate an execution object into containers.

    If an error occurs some containers may have been created and needs to be cleaned-up.
    """
    log.debug('starting all services for execution {}'.format(execution.id))
    execution.set_starting()
    return service_list_to_containers(execution, execution.services)


142
def start_essential(execution: Execution, placement) -> str:
143
144
145
146
    """Start the essential services for this execution"""
    log.debug('starting essential services for execution {}'.format(execution.id))
    execution.set_starting()

147
    return service_list_to_containers(execution, execution.essential_services, placement)
148
149


150
def start_elastic(execution: Execution, placement) -> str:
151
152
    """Start the runnable elastic services"""
    elastic_to_start = [s for s in execution.elastic_services if s.status == Service.RUNNABLE_STATUS]
153
    return service_list_to_containers(execution, elastic_to_start, placement)
154
155


156
157
def terminate_service(service: Service) -> None:
    """Terminate a single service."""
158
    backend = _get_backend()
159
160
161
162
    if service.status != Service.INACTIVE_STATUS:
        if service.status == Service.ERROR_STATUS:
            return
        elif service.status == Service.ACTIVE_STATUS or service.status == Service.TERMINATING_STATUS or service.status == Service.STARTING_STATUS:
163
164
165
166
            service.set_terminating()
            backend.terminate_service(service)
            service.set_inactive()
            log.debug('Service {} terminated'.format(service.name))
167
168
169
170
171
172
173
174
175
176
        elif service.status == Service.CREATED_STATUS or service.status == Service.RUNNABLE_STATUS:
            service.set_inactive()
        else:
            log.error('BUG: don\'t know how to terminate a service in status {}'.format(service.status))
    elif not service.is_dead():
        log.warning('Service {} is inactive for Zoe, but running for the back-end, terminating and resetting state'.format(service.name))
        service.set_terminating()
        backend.terminate_service(service)
        service.set_inactive()
        log.debug('Service {} terminated'.format(service.name))
177
178


179
180
181
182
183
def terminate_execution(execution: Execution) -> None:
    """Terminate an execution."""
    for service in execution.services:  # type: Service
        terminate_service(service)
    execution.set_terminated()
184
185


186
def get_platform_state(state: SQLManager, with_usage_stats=False) -> ClusterStats:
187
    """Retrieves the state of the platform by querying the container backend. Platform state includes information on free/reserved resources for each node. This information is used for advanced scheduling.
188
    Since retrieving usage statistics is slow, you need to ask for them explicitly."""
189
    backend = _get_backend()
190
    platform_state = backend.platform_state(with_usage_stats)
191
    for node in platform_state.nodes:  # type: NodeStats
192
193
        node.services = state.service_list(backend_host=node.name, backend_status=Service.BACKEND_START_STATUS)
    return platform_state
194
195
196
197
198
199
200
201
202
203
204
205


def preload_image(image_name):
    """Make a service image available on the cluster, according to the backend support."""
    backend = _get_backend()
    log.debug('Preloading image {}'.format(image_name))
    time_start = time.time()
    try:
        backend.preload_image(image_name)
        log.info('Image {} preloaded in {:.2f}s'.format(image_name, time.time() - time_start))
    except NotImplementedError:
        log.warning('Backend {} does not support image preloading'.format(get_conf().backend))
206
207
208
209
210
211
212
213
214
215
216
217
218
219


def update_service_resource_limits(service, cores=None, memory=None):
    """Update a service reservation."""
    backend = _get_backend()
    backend.update_service(service, cores, memory)


def node_list():
    """List node names configured in the back-end."""
    backend = _get_backend()
    return backend.node_list()


220
221
222
223
224
225
226
227
228
def node_state(node_name: str, get_usage_stats: bool) -> NodeStats:
    """Get the state of a single node."""
    backend = _get_backend()
    state = SQLManager(get_conf())
    node = backend.node_state(node_name, get_usage_stats)
    node.services = state.service_list(backend_host=node_name, backend_status=Service.BACKEND_START_STATUS)
    return node


229
230
231
232
def list_available_images(node_name):
    """List the images available on the specified node."""
    backend = _get_backend()
    return backend.list_available_images(node_name)