Commit c0841132 authored by Quang-Nhat HOANG-XUAN's avatar Quang-Nhat HOANG-XUAN Committed by GitHub

Merge pull request #79 from DistributedSystemsGroup/kpmg/wp12-proxy

Kpmg/wp12 proxy
parents c8b49392 99e208a2
......@@ -5,7 +5,7 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /media/sdb/kubernetes/data/config:/opt/zoe/kube.conf
command: python3 zoe-api.py --debug --backend Kubernetes --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --master-url tcp://zoe-master:4850 --auth-type text --proxy-type none --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5001
command: python3 zoe-api.py --debug --backend Kubernetes --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --master-url tcp://zoe-master:4850 --auth-type text --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5001
ports:
- "5001:5001"
zoe-master:
......@@ -14,7 +14,7 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- /mnt/zoe-workspaces:/mnt/zoe-workspaces
- /media/sdb/kubernetes/data/config:/opt/zoe/kube.conf
command: python3 zoe-master.py --debug --backend Kubernetes --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --auth-type text --proxy-type none --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5001
command: python3 zoe-master.py --debug --backend Kubernetes --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --auth-type text --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5001
ports:
- "4850:4850"
depends_on:
......
......@@ -4,7 +4,7 @@ services:
image: 192.168.12.2:5000/zoe:reducetime
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: python3 zoe-api.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master:4850 --auth-type ldap --ldap-server-uri ldap://172.17.0.6 --ldap-base-dn ou=users,dc=example,dc=com --proxy-type apache --proxy-container apache2 --proxy-config-file /etc/apache2/sites-available/all.conf --proxy-path fsdna.on.kpmg.de/zoe
command: python3 zoe-api.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master:4850 --auth-type ldap --ldap-server-uri ldap://172.17.0.6 --ldap-base-dn ou=users,dc=example,dc=com --proxy-path zoe.fsdna.on.kpmg.de
ports:
- "5001:5001"
logging:
......@@ -16,7 +16,7 @@ services:
image: 192.168.12.2:5000/zoe:reducetime
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: python3 zoe-master.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type ldap --ldap-server-uri ldap://ldapker.example.com --ldap-base-dn ou=users,dc=example,dc=com --proxy-type apache --proxy-container apache2 --proxy-config-file /etc/apache2/sites-available/all.conf --proxy-path fsdna.on.kpmg.de/zoe
command: python3 zoe-master.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name prod --dbuser postgres --dbhost 192.168.12.2 --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type ldap --ldap-server-uri ldap://ldapker.example.com --ldap-base-dn ou=users,dc=example,dc=com --proxy-path zoe.fsdna.on.kpmg.de
ports:
- "4850:4850"
depends_on:
......
......@@ -5,7 +5,7 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /media/sdb/kubernetes/data/config:/opt/zoe/kube.conf
command: python3 zoe-api.py --debug --backend Kubernetes --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --proxy-type none --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5100
command: python3 zoe-api.py --debug --backend Kubernetes --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5100
ports:
- "5100:5100"
zoe-master-test:
......@@ -14,6 +14,6 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- /mnt/zoe-workspaces:/mnt/zoe-workspaces
- /media/sdb/kubernetes/data/config:/opt/zoe/kube.conf
command: python3 zoe-master.py --debug --backend Kubernetes --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text --proxy-type none --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5100
command: python3 zoe-master.py --debug --backend Kubernetes --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text --proxy-path zoe.fsdna.on.kpmg.de --listen-port 5100
depends_on:
- zoe-api-test
......@@ -4,7 +4,7 @@ services:
image: 192.168.12.2:5000/zoe:wi9
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: python3 zoe-api.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --proxy-type none --listen-port 5100
command: python3 zoe-api.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --listen-port 5100
ports:
- "5100:5100"
zoe-master-test:
......@@ -12,6 +12,6 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /mnt/zoe-workspaces:/mnt/zoe-workspaces
command: python3 zoe-master.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text --proxy-type none --listen-port 5100
command: python3 zoe-master.py --debug --backend-swarm-url consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text --listen-port 5100
depends_on:
- zoe-api-test
.. _proxy:
Access ZApps through proxy
==========================
Access ZApps through Ingress Controller on Kubernetes
=====================================================
Overview
--------
* We can access Zapps through a web proxy, so we do not need to open too many ports due to security reasons.
* The Zapps will be preconfigured the ``base_url`` or ``context_path`` when building image.
* By using a web proxy, services which are exposed in Zapp can be access through the proxy url, which has the following format: ``base_proxy_path/zoe/userID/executionID/serviceName``
* This can be achieved when Zoe runs on Kubernetes by the support of an Ingress Controller.
* Automate the process of creating an ingress for a servive created by Zoe.
* Services which are exposed in Zapp can be accessed through the proxy url, which has the following format: ``http://servicename-executionid-deploymentname.proxy-path``
Requirements
------------
* A web proxy. We use **Apache2** as the default one.
* The service which is needed to be accessed through web proxy needs to support running in ``base_url`` or ``context_path`` and is defined as expose: True in the zap description.
* Specify the environment key, value in zapp description to specify the base_url, the value will be overridden by Zoe to assure the format of the proxy url.
* A Kubernetes cluster which has:
* Zoe
* A (NGINX) ingress controller.
* kubernetes-auto-ingress.
How it works
------------
1. Zoe configuration file:
* ``--proxy-type``: web proxy type to be used, default is Apache, Nginx can be added in future
* ``--proxy-container``: the container which we run the proxy web server
* ``--proxy-config-file``: the configuration (VirtualHost configuration file) file path of the proxy web server
* ``--proxy-path``: the **ServerName** field in apache2 virtualhost configuration
2. Proxy server:
* Apache web server is running inside a container, default is named as ``apache2``
* Apache configuration file inside ``/apache2_installation_path/sites-available``, default is ``zoe.conf``
* ``mod_proxy``, ``modproxyhttp`` much be enabled
3. Zoe:
* Services supports to configure ``base_url`` via environment variables, Zoe will get those environment variables and generate the proxy path for each service which is defined as ``expose: True``, then passes them as containers’ options to create containers.
* When **starting a new execution**, after all services have **started** states, Zoe begins to **proxify** them. It will:
* Connect to the ``apache2`` container and adds new entries to the ``zoe.conf`` these lines:
2. (NGINX) ingress controller:
* ``ProxyPass /base_proxy_path/zoe/userID/executionID/serviceName http://ip:port``
* ``ProxyPassReverse /base_proxy_path/zoe/userID/executionID/serviceName http://ip:port``
* Which:
* An Ingress is a collection of rules that allow inbound connections to reach the cluster services.
* In order for the Ingress resource to work, the cluster must have an Ingress controller running. The Ingress controller will manage, configure the description in the Ingress resource to expose the associated services.
* IP: IP of the host which the container is running on
* Port: the binding port at the host
* Reload the ``apache2`` service running inside the ``apache2`` container
* When terminating an execution, the **unproxify** process happens, which is similar to the proxifying process. Instead of inserting new entries, it deletes entries which corresponding to the terminating execution.
3. kubernetes-auto-ingress:
Note
----
* The proxifying and unproxifying process can be done manually: you can go directly to the proxy server container and add entries into the configuration file then reload the service
* The current proxy server we use is Apache2. Nginx proxy server can be added in future with the same idea.
* Currently, the process to submit an Ingress resource to the Ingress controller is mannually done by cluster admins. kubernetes-auto-ingress automates this process. Every services have the labels "auto-ingress/enabled" is "enabled" will be automatically attached with the associated ingress resources.
References
----------
* Apache web server: https://httpd.apache.org/
* Apache reverse proxy: https://httpd.apache.org/docs/2.4/mod/mod_proxy.html
* Apache virtualhost example: https://httpd.apache.org/docs/2.4/vhosts/examples.html
* Kubernetes Ingress: https://kubernetes.io/docs/concepts/services-networking/ingress/
* NGINX Ingress Controller: https://github.com/kubernetes/ingress/tree/master/controllers/nginx
* kubernetes-auto-ingress: https://github.com/hxquangnhat/kubernetes-auto-ingress
......@@ -17,12 +17,9 @@
import logging
import re
import threading
import zoe_api.exceptions
import zoe_api.master_api
from zoe_api.proxy.apache import ApacheProxy
#from zoe_api.proxy.nginx import NginxProxy
import zoe_lib.applications
import zoe_lib.exceptions
import zoe_lib.state
......@@ -58,16 +55,6 @@ class APIEndpoint:
ret = [e for e in execs if e.user_id == uid or role == 'admin']
return ret
def _get_proxy(self):
if get_conf().proxy_type == 'apache':
proxy = ApacheProxy(self)
# elif get_conf().proxy_type == 'nginx':
# proxy = NginxProxy(self)
else:
log.info('Proxy disabled')
proxy = None
return proxy
def zapp_validate(self, application_description):
"""Validates the passed ZApp description against the supported schema."""
try:
......@@ -75,7 +62,7 @@ class APIEndpoint:
except zoe_lib.exceptions.InvalidApplicationDescription as e:
raise zoe_api.exceptions.ZoeException('Invalid application description: ' + e.message)
def execution_start(self, uid, role, exec_name, application_description):
def execution_start(self, uid, role, exec_name, application_description): # pylint: disable=unused-argument
"""Start an execution."""
try:
zoe_lib.applications.app_validate(application_description)
......@@ -92,10 +79,6 @@ class APIEndpoint:
if not success:
raise zoe_api.exceptions.ZoeException('The Zoe master is unavailable, execution will be submitted automatically when the master is back up ({}).'.format(message))
proxy = self._get_proxy()
if proxy is not None:
threading.Thread(target=proxy.proxify, args=(uid, role, new_id)).start()
return new_id
def execution_terminate(self, uid, role, exec_id):
......@@ -109,9 +92,6 @@ class APIEndpoint:
raise zoe_api.exceptions.ZoeAuthException()
if e.is_active:
proxy = self._get_proxy()
if proxy is not None:
proxy.unproxify(uid, role, exec_id)
return self.master.execution_terminate(exec_id)
else:
raise zoe_api.exceptions.ZoeException('Execution is not running')
......
# Copyright (c) 2016, Quang-Nhat Hoang-Xuan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Proxifying using Apache2 Container."""
import time
import logging
import random
import docker
import zoe_api.proxy.base
import zoe_api.api_endpoint
from zoe_master.backends.swarm.api_client import SwarmClient
from zoe_master.backends.kubernetes.api_client import KubernetesClient
from zoe_lib.config import get_conf
log = logging.getLogger(__name__)
class ApacheProxy(zoe_api.proxy.base.BaseProxy):
"""Apache proxy class."""
def __init__(self, api_endpoint):
self.api_endpoint = api_endpoint
def proxify(self, uid, role, execution_id): # pylint: disable=too-many-locals
"""Proxify function."""
try:
# Wait until all the services get created and started to be able to get the backend_id
while self.api_endpoint.execution_by_id(uid, role, execution_id).status != 'running':
log.info('Waiting for all services get started...')
time.sleep(1)
exe = self.api_endpoint.execution_by_id(uid, role, execution_id)
lth = len(exe.services)
while lth != 0:
exe = self.api_endpoint.execution_by_id(uid, role, execution_id)
lth = len(exe.services)
for srv in exe.services:
if srv.backend_id is None:
time.sleep(1)
else:
lth -= 1
# Start proxifying by adding entry to use proxypass and proxypassreverse in apache2 config file
for srv in exe.services:
ip, port = None, None
if get_conf().backend == 'OldSwarm':
swarm = SwarmClient()
s_info = swarm.inspect_container(srv.backend_id)
port_list = s_info['ports']
for key, val in port_list.items():
exposed_port = key.split('/tcp')[0]
if val is not None:
ip = val[0]
port = val[1]
base_path = '/zoe/' + uid + '/' + str(execution_id) + '/' + srv.name + '/' + exposed_port
original_path = str(ip) + ':' + str(port) + base_path
if ip is not None and port is not None:
log.info('Proxifying %s', srv.name + ' port ' + exposed_port)
self.dispatch_to_docker(base_path, original_path)
else:
kube = KubernetesClient(get_conf())
s_info = kube.inspect_service(srv.dns_name)
kube_nodes = kube.info().nodes
host_ip = random.choice(kube_nodes).name
while 'nodePort' not in s_info['port_forwarding'][0]:
log.info('Waiting for service get started before proxifying...')
s_info = kube.inspect_service(srv.dns_name)
time.sleep(0.5)
ip = host_ip
port = s_info['port_forwarding'][0]['nodePort']
exposed_port = s_info['port_forwarding'][0]['port']
base_path = '/zoe/' + uid + '/' + str(execution_id) + '/' + srv.name + '/' + str(exposed_port)
original_path = str(ip) + ':' + str(port) + base_path
if ip is not None and port is not None:
log.info('Proxifying %s', srv.name + ' port ' + str(exposed_port))
self.dispatch_to_docker(base_path, original_path)
except Exception as ex:
log.error(ex)
def dispatch_to_docker(self, base_path, original_path):
"""
The apache2 server is running inside a container
Adding new entries with the proxy path and the ip:port of the application to the apache2 config file
"""
proxy = ['ProxyPass ' + base_path + '/api/kernels/ ws://' + original_path + '/api/kernels/',
'ProxyPassReverse ' + base_path + '/api/kernels/ ws://' + original_path + '/api/kernels/',
'ProxyPass ' + base_path + '/terminals/websocket/ ws://' + original_path + '/terminals/websocket/',
'ProxyPassReverse ' + base_path + '/terminals/websocket/ ws://' + original_path + '/terminals/websocket/',
'ProxyPass ' + base_path + ' http://' + original_path,
'ProxyPassReverse ' + base_path + ' http://' + original_path,
'',
'</VirtualHost>']
docker_client = docker.DockerClient(base_url=get_conf().proxy_docker_sock)
del_command = "sed -i '$ d' " + get_conf().proxy_config_file # /etc/apache2/sites-available/all.conf"
proxy_container = docker_client.containers.get(get_conf().proxy_container)
proxy_container.exec_run(del_command)
for entry in proxy:
command = 'bash -c "echo ' + "'" + entry + "'" + ' >> /etc/apache2/sites-available/all.conf"'
proxy_container.exec_run(command)
reload_command = 'service apache2 reload'
proxy_container.exec_run(reload_command)
# Simply remove the added entries at the apache2 config file when terminating applications
def unproxify(self, uid, role, execution_id):
"""Un-proxify."""
log.info('Unproxifying for user %s - execution %s', uid, str(execution_id))
pattern = '/zoe\/' + uid + '\/' + str(execution_id) + '/d' # pylint: disable=anomalous-backslash-in-string
docker_client = docker.DockerClient(base_url=get_conf().proxy_docker_sock)
del_command = 'sed -i "' + pattern + '" ' + get_conf().proxy_config_file # /etc/apache2/sites-available/all.conf'
proxy_container = docker_client.containers.get(get_conf().proxy_container)
proxy_container.exec_run(del_command)
# Copyright (c) 2016, Quang-Nhat Hoang-Xuan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base Proxy class."""
class BaseProxy:
"""Base proxy class."""
def proxify(self, uid, role, execution_id):
"""The methods that needs to be overridden by implementations."""
raise NotImplementedError
def unproxify(self, uid, role, execution_id):
"""The methods that needs to be overridden by implementations."""
raise NotImplementedError
......@@ -39,7 +39,7 @@
{% if s['backend_status'] == 'started' %}
{% for p in s['description']['ports'] %}
{% if s['proxy_address'] is not none %}
<li><a href="{{ p['protocol'] }}://{{ s['proxy_address'] }}/{{ p['port_number'] }}">{{ p['name'] }}</a></li>
<li><a href="http://{{ s['proxy_address'] }}">{{ p['name'] }}</a></li>
{% else %}
<li><a> {{ p['name'] }} IP: {{ s['ip_address'] }}</a></li>
{% endif %}
......
......@@ -83,11 +83,7 @@ def load_configuration(test_conf=None):
argparser.add_argument('--ldap-guest-gid', type=int, help='LDAP group ID for guests', default=5002)
# Proxy options
argparser.add_argument('--proxy-type', help='Proxy type (apache or nginx)', default='none')
argparser.add_argument('--proxy-container', help='Proxy container name', default='apache2')
argparser.add_argument('--proxy-config-file', help='Config file path of apache/nginx proxy container', default='/etc/apache2/sites-available/config.conf')
argparser.add_argument('--proxy-path', help='Proxy base path', default='127.0.0.1')
argparser.add_argument('--proxy-docker-sock', help='Docker sock url which proxy container uses', default='unix://var/run/docker.sock')
argparser.add_argument('--scheduler-class', help='Scheduler class to use for scheduling ZApps', choices=['ZoeSimpleScheduler', 'ZoeElasticScheduler'], default='ZoeSimpleScheduler')
argparser.add_argument('--scheduler-policy', help='Scheduler policy to use for scheduling ZApps', choices=['FIFO', 'SIZE'], default='FIFO')
......@@ -103,6 +99,7 @@ def load_configuration(test_conf=None):
# Kubernetes backend
argparser.add_argument('--kube-config-file', help='Kubernetes configuration file', default='/opt/zoe/kube.conf')
argparser.add_argument('--kube-namespace', help='The namespace that Zoe operates on', default='default')
argparser.add_argument('--cookie-secret', help='secret used to encrypt cookies', default='changeme')
argparser.add_argument('--log-file', help='output logs to a file', default='stderr')
......
......@@ -193,7 +193,7 @@ class Service:
def proxy_address(self):
"""Get proxy address path"""
if len(self.ports) > 0:
return get_conf().proxy_path + "/" + self.user_id + "/" + str(self.execution_id) + "/" + self.name
return self.name + "-" + str(self.execution_id) + "-" + get_conf().deployment_name + "." + get_conf().proxy_path
else:
return None
......
......@@ -19,7 +19,6 @@ from typing import Dict
from zoe_lib.config import get_conf
from zoe_lib.state import Service, Execution
from zoe_master.backends.proxy import gen_proxypath, JUPYTER_NOTEBOOK, MONGO_EXPRESS, JUPYTER_PORT, MONGO_PORT
from zoe_master.exceptions import ZoeStartExecutionFatalException
from zoe_master.workspace.filesystem import ZoeFSWorkspace
......@@ -38,12 +37,6 @@ def gen_environment(execution: Execution, service: Service, env_subst_dict: Dict
raise ZoeStartExecutionFatalException("Service {} has wrong environment expression")
env_list.append((env_name, env_value))
# FIXME this code needs to be removed/changed to be generic
#if 'jupyter' in service.image_name:
env_list.append((JUPYTER_NOTEBOOK, gen_proxypath(execution, service) + '/' + JUPYTER_PORT))
#elif 'mongo-express' in service.image_name:
env_list.append((MONGO_EXPRESS, gen_proxypath(execution, service) + '/' + MONGO_PORT))
env_list.append(('EXECUTION_ID', str(execution.id)))
env_list.append(('DEPLOY_NAME', get_conf().deployment_name))
env_list.append(('UID', execution.user_id))
......
......@@ -27,15 +27,16 @@ from zoe_master.stats import ClusterStats, NodeStats
from zoe_master.backends.service_instance import ServiceInstance
from zoe_lib.version import ZOE_VERSION
from zoe_lib.state import VolumeDescription
from zoe_lib.config import get_conf
log = logging.getLogger(__name__)
ZOE_LABELS = {
"app": "zoe",
"version": ZOE_VERSION
"version": ZOE_VERSION,
"auto-ingress/enabled" : "enabled"
}
class KubernetesConf:
"""Kubeconfig class"""
def __init__(self, jsonfile):
......@@ -51,7 +52,8 @@ class KubernetesServiceConf:
'kind': 'Service',
'apiVersion': "v1",
'metadata': {
'labels': {}
'labels': {},
'namespace': get_conf().kube_namespace
},
'spec': {
'selector': {},
......@@ -97,7 +99,8 @@ class KubernetesReplicationControllerConf:
'kind': 'ReplicationController',
'apiVersion': "v1",
'metadata': {
'labels': {}
'labels': {},
'namespace': get_conf().kube_namespace
},
'spec': {
'replicas': 1,
......@@ -265,7 +268,7 @@ class KubernetesClient:
def inspect_replication_controller(self, name):
"""Get information about a specific replication controller."""
try:
repcon_list = pykube.ReplicationController.objects(self.api)
repcon_list = pykube.ReplicationController.objects(self.api).filter(namespace=get_conf().kube_namespace)
rep = repcon_list.get_by_name(name)
rc_info = rep.obj
......@@ -304,7 +307,7 @@ class KubernetesClient:
def replication_controller_list(self):
"""Get list of replication controller."""
repcon_list = pykube.ReplicationController.objects(self.api).filter(selector=ZOE_LABELS).iterator()
repcon_list = pykube.ReplicationController.objects(self.api).filter(namespace=get_conf().kube_namespace, selector=ZOE_LABELS).iterator()
rclist = []
try:
for rep in repcon_list:
......@@ -313,11 +316,6 @@ class KubernetesClient:
log.error(ex)
return rclist
def replication_controller_event(self):
"""Get event stream of the replication controller."""
rc_stream = pykube.ReplicationController.objects(self.api).filter(selector=ZOE_LABELS).watch()
return rc_stream
def spawn_service(self, service_instance: ServiceInstance):
"""Create and start a new Service object."""
config = KubernetesServiceConf()
......@@ -342,7 +340,7 @@ class KubernetesClient:
def inspect_service(self, name) -> Dict[str, Any]:
"""Get information of a specific service."""
try:
service_list = pykube.Service.objects(self.api)
service_list = pykube.Service.objects(self.api).filter(namespace=get_conf().kube_namespace)
service = service_list.get_by_name(name)
srv_info = service.obj
......@@ -374,7 +372,8 @@ class KubernetesClient:
'apiVersion': 'v1',
'kind': '',
'metadata': {
'name': name
'name': name,
'namespace': get_conf().kube_namespace
}
}
try:
......@@ -387,7 +386,7 @@ class KubernetesClient:
del_obj['kind'] = 'Pod'
pod_selector = ZOE_LABELS
pod_selector['service_name'] = name
pods = pykube.Pod.objects(self.api).filter(namespace="default", selector=pod_selector).iterator()
pods = pykube.Pod.objects(self.api).filter(namespace=get_conf().kube_namespace, selector=pod_selector).iterator()
for pod in pods:
del_obj['metadata']['name'] = str(pod)
pykube.Pod(self.api, del_obj).delete()
......@@ -400,7 +399,7 @@ class KubernetesClient:
"""Retrieve Kubernetes cluster statistics."""
pl_status = ClusterStats()
node_list = pykube.Node.objects(self.api).iterator()
node_list = pykube.Node.objects(self.api).filter(namespace=pykube.all).iterator()
node_dict = {}
# Get basic information from nodes
......
......@@ -18,6 +18,7 @@
import logging
import threading
import time
import pykube
from zoe_lib.config import get_conf
from zoe_lib.state import SQLManager, Service
......@@ -43,38 +44,43 @@ class KubernetesMonitor(threading.Thread):
"""An infinite loop that listens for events from Kubernetes."""
log.info("Monitor thread started")
while True: # pylint: disable=too-many-nested-blocks
for event in self.kube.replication_controller_event():
log.debug('%s: %s', event.object.name, event.type)
if event.type != 'DELETED' and event.type != 'ADDED':
rc_info = self.kube.inspect_replication_controller(event.object.name)
if rc_info:
rc_uid = rc_info['backend_id']
service = self.state.service_list(only_one=True, backend_id=rc_uid)
if event.object.name not in self.service_id:
self.service_id[event.object.name] = service.id
if service is not None:
if rc_info['readyReplicas'] == 0:
log.debug('Number replicas: 0')
service.set_backend_status(service.BACKEND_UNDEFINED_STATUS)
elif rc_info['readyReplicas'] < rc_info['replicas']:
logstr = 'Number replicas: ' + str(rc_info['readyReplicas'])
log.debug(logstr)
service.set_backend_status(service.BACKEND_CREATE_STATUS)
elif rc_info['readyReplicas'] == rc_info['replicas']:
if service.backend_status != service.BACKEND_START_STATUS:
log.debug('Reached desired number of replicas')
service.set_backend_status(service.BACKEND_START_STATUS)
else:
if event.type != 'ADDED':
if event.object.name in self.service_id:
sid = self.service_id[event.object.name]
self.service_id.pop(event.object.name)
service = self.state.service_list(only_one=True, id=sid)
log.debug("Kubernetes service event stream")
try:
watch = pykube.ReplicationController.objects(self.kube.api, namespace=get_conf().kube_namespace).watch()
for event in watch:
log.debug('%s: %s', event.object.name, event.type)
if event.type != 'DELETED' and event.type != 'ADDED':
rc_info = self.kube.inspect_replication_controller(event.object.name)
if rc_info:
rc_uid = rc_info['backend_id']
service = self.state.service_list(only_one=True, backend_id=rc_uid)
if event.object.name not in self.service_id:
self.service_id[event.object.name] = service.id
if service is not None:
log.info('Destroyed all replicas')
service.set_backend_status(service.BACKEND_DESTROY_STATUS)
time.sleep(1)
if rc_info['readyReplicas'] == 0:
log.debug('Number replicas: 0')
service.set_backend_status(service.BACKEND_UNDEFINED_STATUS)
elif rc_info['readyReplicas'] < rc_info['replicas']:
logstr = 'Number replicas: ' + str(rc_info['readyReplicas'])
log.debug(logstr)
service.set_backend_status(service.BACKEND_CREATE_STATUS)
elif rc_info['readyReplicas'] == rc_info['replicas']:
if service.backend_status != service.BACKEND_START_STATUS:
log.debug('Reached desired number of replicas')
service.set_backend_status(service.BACKEND_START_STATUS)
else:
if event.type != 'ADDED':
if event.object.name in self.service_id:
sid = self.service_id[event.object.name]
self.service_id.pop(event.object.name)
service = self.state.service_list(only_one=True, id=sid)
if service is not None:
log.info('Destroyed all replicas')
service.set_backend_status(service.BACKEND_DESTROY_STATUS)
time.sleep(1)
except Exception as ex:
log.error(ex)
log.debug("Kubernetes service event stream ended, start new stream")
time.sleep(2)
def quit(self):
......@@ -101,6 +107,7 @@ class KubernetesStateSynchronizer(threading.Thread):
"""Loop through the pods and try to update the service status."""
found = False
for rep in repcon_list:
log.debug("%s - %s", rep['backend_id'], service.backend_id)
if rep['backend_id'] == service.backend_id:
found = True
if rep['running'] is False:
......
# Copyright (c) 2017, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The high-level interface that Zoe uses to talk to the configured container backend."""
from zoe_lib.state import Service, Execution
JUPYTER_NOTEBOOK = 'BASE_URL'
MONGO_EXPRESS = 'ME_CONFIG_SITE_BASEURL'
JUPYTER_PORT = '8888'
MONGO_PORT = '27017'
def gen_proxypath(execution: Execution, service: Service):
""" gen proxy address path """
proxy_path_value = '/zoe/' + execution.user_id + '/' + str(execution.id) + '/' + service.name
return proxy_path_value
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment