Commit cb70f515 authored by Daniele Venzano's avatar Daniele Venzano

Rename the scheduler component into the Master component.

parent d53b79cb
......@@ -22,9 +22,9 @@ Repository contents
- `docs`: Sphinx documentation
- `scripts`: Scripts used to test Zoe images outside of Zoe
- `zoe_cmd`: Command-line client
- `zoe_lib`: Client-side library, contains also some modules needed by the observer and the scheduler processes
- `zoe_lib`: Client-side library, contains also some modules needed by the observer and the master processes
- `zoe_logger`: Optional Kafka producer for Docker container logs
- `zoe_observer`: The Observer process that monitors Swarm and informs the scheduler of various events
- `zoe_observer`: The Observer process that monitors Swarm and informs the master of various events
- `zoe_scheduler`: The core of Zoe, the server process that listens for client requests and creates the containers on Swarm
- `zoe_web`: The web client interface
......
[program:zoe-scheduler]
command=/usr/bin/python3 /home/ubuntu/zoe/zoe-scheduler.py
[program:zoe-master]
command=/usr/bin/python3 /home/ubuntu/zoe/zoe-master.py
directory=/home/ubuntu/zoe
autostart=true
autorestart=true
startretries=3
stderr_logfile=/var/log/zoe/scheduler.err.log
stdout_logfile=/var/log/zoe/scheduler.out.log
stderr_logfile=/var/log/zoe/master.err.log
stdout_logfile=/var/log/zoe/master.out.log
user=ubuntu
[program:zoe-web]
......
Scheduler classes
=================
.. autoclass:: zoe_scheduler.scheduler.ZoeScheduler
.. autoclass:: zoe_master.scheduler.ZoeScheduler
:members:
.. autoclass:: zoe_scheduler.scheduler_policies.base.BaseSchedulerPolicy
.. autoclass:: zoe_master.scheduler_policies.base.BaseSchedulerPolicy
:members:
......@@ -64,7 +64,7 @@ Currently this is the recommended procedure:
1. Clone the zoe repository
2. Install Python package dependencies: ``pip3 install -r requirements.txt``
3. Create new configuration files for the scheduler and the observer (:ref:`config_file`)
3. Create new configuration files for the master and the observer (:ref:`config_file`)
4. Setup supervisor to manage Zoe processes: in the ``scripts/supervisor/`` directory you can find the configuration file for
supervisor. You need to modify the paths to point to where you cloned Zoe and the user (Zoe does not need special privileges).
5. Start running applications using the command-line client! (the web interface will be coming soon)
......@@ -9,5 +9,5 @@ fi
SWARM_ADDRESS="swarm:2380"
sudo docker -H ${SWARM_ADDRESS} run -i -t --rm=true -e ZOE_SCHEDULER_SWARM=${SWARM_ADDRESS} zoerepo/zoe-scheduler
sudo docker -H ${SWARM_ADDRESS} run -i -t --rm=true zoerepo/zoe-client ./zoe-web.py
\ No newline at end of file
sudo docker -H ${SWARM_ADDRESS} run -i -t --rm=true -e ZOE_MASTER_SWARM=${SWARM_ADDRESS} zoerepo/zoe-master
sudo docker -H ${SWARM_ADDRESS} run -i -t --rm=true zoerepo/zoe-client ./zoe-web.py
......@@ -24,8 +24,8 @@ from zoe_lib.exceptions import ZoeException
log = logging.getLogger('ZoeAIO')
def start_sched():
from zoe_scheduler.entrypoint import main
def start_master():
from zoe_master.entrypoint import main
main()
......@@ -33,7 +33,7 @@ def start_observer():
from zoe_observer.entrypoint import main
main()
sched = multiprocessing.Process(target=start_sched, name='scheduler')
sched = multiprocessing.Process(target=start_master(), name='master')
obs = multiprocessing.Process(target=start_observer, name='observer')
sched.start()
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.entrypoint import main
from zoe_master.entrypoint import main
if __name__ == '__main__':
main()
......@@ -26,7 +26,7 @@ log = logging.getLogger(__name__)
class ZoeContainerAPI(ZoeAPIBase):
"""
The container API class. Containers are read-only objects. The delete operation merely informs the scheduler that a container has died outside of its control.
The container API class. Containers are read-only objects. The delete operation merely informs the master that a container has died outside of its control.
"""
def get(self, container_id: int) -> dict:
"""
......@@ -81,7 +81,7 @@ class ZoeContainerAPI(ZoeAPIBase):
def died(self, container_id: int):
"""
Inform the scheduler that a container died. Used by the observer process.
Inform the master that a container died. Used by the observer process.
:param container_id: Zoe ID of the container that died
:return:
......
......@@ -58,7 +58,7 @@ class ZoeExecutionsAPI(ZoeAPIBase):
"""
Retrieve the Execution object for an existing execution.
:param execution_id: the execution to load from the scheduler
:param execution_id: the execution to load from the master
:return: the Execution object, or None
"""
data, status_code = self._rest_get('/execution/' + str(execution_id))
......@@ -69,7 +69,7 @@ class ZoeExecutionsAPI(ZoeAPIBase):
def execution_start(self, name: str, application_description: dict) -> int:
"""
Submit an application to the scheduler to start a new execution.
Submit an application to the master to start a new execution.
:param name: user-provided name of the execution
:param application_description: the application to start
......
......@@ -28,7 +28,7 @@ import docker
import docker.errors
import docker.utils
from zoe_scheduler.stats import SwarmStats, SwarmNodeStats, ContainerStats
from zoe_master.stats import SwarmStats, SwarmNodeStats, ContainerStats
from zoe_lib.exceptions import ZoeException
log = logging.getLogger(__name__)
......
......@@ -16,8 +16,8 @@
from zoe_lib.configargparse import ArgumentParser, Namespace
config_paths = [
'zoe-scheduler.conf',
'/etc/zoe/zoe-scheduler.conf'
'zoe-master.conf',
'/etc/zoe/zoe-master.conf'
]
singletons = {
......
......@@ -21,21 +21,21 @@ from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.scheduler_policies import FIFOSchedulerPolicy
import zoe_scheduler.config as config
from zoe_scheduler.rest_api import init as api_init
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.state.blobs.fs import FSBlobs
from zoe_master.platform_manager import PlatformManager
from zoe_master.scheduler_policies import FIFOSchedulerPolicy
import zoe_master.config as config
from zoe_master.rest_api import init as api_init
from zoe_master.state.manager import StateManager
from zoe_master.state.blobs.fs import FSBlobs
from zoe_lib.metrics.influxdb import InfluxDBMetricSender
from zoe_scheduler.stats_manager import StatsManager
from zoe_master.stats_manager import StatsManager
log = logging.getLogger("main")
def main():
"""
The entrypoint for the zoe-scheduler script.
The entrypoint for the zoe-master script.
:return: int
"""
config.load_configuration()
......
......@@ -18,12 +18,12 @@ import logging
from zoe_lib.swarm_client import SwarmClient, ContainerOptions
from zoe_lib.exceptions import ZoeException
from zoe_scheduler.config import get_conf, singletons
from zoe_scheduler.scheduler import ZoeScheduler
from zoe_scheduler.state import execution as execution_module, application as application_module, container as container_module
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.stats import ContainerStats
from zoe_scheduler.stats import SwarmStats, SchedulerStats
from zoe_master.config import get_conf, singletons
from zoe_master.scheduler import ZoeScheduler
from zoe_master.state import execution as execution_module, application as application_module, container as container_module
from zoe_master.state.manager import StateManager
from zoe_master.stats import ContainerStats
from zoe_master.stats import SwarmStats, SchedulerStats
log = logging.getLogger(__name__)
......
......@@ -17,11 +17,11 @@ import sys
from flask import Flask
from flask_restful import Api
from zoe_scheduler.rest_api.user import UserAPI, UserCollectionAPI
from zoe_scheduler.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
from zoe_scheduler.rest_api.container import ContainerAPI
from zoe_scheduler.rest_api.query import QueryAPI
from zoe_scheduler.rest_api.info import InfoAPI
from zoe_master.rest_api.user import UserAPI, UserCollectionAPI
from zoe_master.rest_api.execution import ExecutionAPI, ExecutionCollectionAPI
from zoe_master.rest_api.container import ContainerAPI
from zoe_master.rest_api.query import QueryAPI
from zoe_master.rest_api.info import InfoAPI
from zoe_lib.version import ZOE_API_VERSION
API_PATH = '/api/' + ZOE_API_VERSION
......
......@@ -18,12 +18,12 @@ import time
from flask_restful import Resource, request
from zoe_lib.exceptions import ZoeRestAPIException
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.rest_api.utils import catch_exceptions
from zoe_scheduler.rest_api.auth.authentication import authenticate
from zoe_scheduler.rest_api.auth.authorization import is_authorized
from zoe_scheduler.config import singletons
from zoe_master.state.manager import StateManager
from zoe_master.platform_manager import PlatformManager
from zoe_master.rest_api.utils import catch_exceptions
from zoe_master.rest_api.auth.authentication import authenticate
from zoe_master.rest_api.auth.authorization import is_authorized
from zoe_master.config import singletons
log = logging.getLogger(__name__)
......
......@@ -18,13 +18,13 @@ from werkzeug.exceptions import BadRequest
from flask_restful import Resource, request
from zoe_lib.exceptions import ZoeException, ZoeRestAPIException
from zoe_scheduler.rest_api.utils import catch_exceptions
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.state.execution import Execution
from zoe_scheduler.rest_api.auth.authentication import authenticate
from zoe_scheduler.rest_api.auth.authorization import is_authorized, check_quota
from zoe_scheduler.config import singletons
from zoe_master.rest_api.utils import catch_exceptions
from zoe_master.state.manager import StateManager
from zoe_master.platform_manager import PlatformManager
from zoe_master.state.execution import Execution
from zoe_master.rest_api.auth.authentication import authenticate
from zoe_master.rest_api.auth.authorization import is_authorized, check_quota
from zoe_master.config import singletons
class ExecutionAPI(Resource):
......
......@@ -18,11 +18,11 @@ import time
from flask_restful import Resource, request
from zoe_lib.version import ZOE_API_VERSION, ZOE_APPLICATION_FORMAT_VERSION, ZOE_VERSION
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.rest_api.utils import catch_exceptions
from zoe_scheduler.rest_api.auth.authentication import authenticate
from zoe_scheduler.config import singletons, get_conf
from zoe_master.state.manager import StateManager
from zoe_master.platform_manager import PlatformManager
from zoe_master.rest_api.utils import catch_exceptions
from zoe_master.rest_api.auth.authentication import authenticate
from zoe_master.config import singletons, get_conf
class InfoAPI(Resource):
......
......@@ -19,11 +19,11 @@ from werkzeug.exceptions import BadRequest
from flask_restful import Resource, request
from zoe_lib.exceptions import ZoeRestAPIException
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.rest_api.utils import catch_exceptions
from zoe_scheduler.rest_api.auth.authentication import authenticate
from zoe_scheduler.config import singletons
from zoe_master.state.manager import StateManager
from zoe_master.platform_manager import PlatformManager
from zoe_master.rest_api.utils import catch_exceptions
from zoe_master.rest_api.auth.authentication import authenticate
from zoe_master.config import singletons
class QueryAPI(Resource):
......
......@@ -19,13 +19,13 @@ from werkzeug.exceptions import BadRequest
from flask_restful import Resource, request
from zoe_lib.exceptions import ZoeException, ZoeRestAPIException
from zoe_scheduler.rest_api.utils import catch_exceptions
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.platform_manager import PlatformManager
from zoe_scheduler.rest_api.auth.authentication import authenticate
from zoe_scheduler.rest_api.auth.authorization import is_authorized
from zoe_scheduler.state.user import User
from zoe_scheduler.config import singletons
from zoe_master.rest_api.utils import catch_exceptions
from zoe_master.state.manager import StateManager
from zoe_master.platform_manager import PlatformManager
from zoe_master.rest_api.auth.authentication import authenticate
from zoe_master.rest_api.auth.authorization import is_authorized
from zoe_master.state.user import User
from zoe_master.config import singletons
class UserAPI(Resource):
......
......@@ -17,9 +17,9 @@ import logging
import queue
from zoe_lib.exceptions import ZoeException
from zoe_scheduler.state.application import Application
from zoe_scheduler.state.execution import Execution
from zoe_scheduler.scheduler_policies.base import BaseSchedulerPolicy
from zoe_master.state.application import Application
from zoe_master.state.execution import Execution
from zoe_master.scheduler_policies.base import BaseSchedulerPolicy
log = logging.getLogger(__name__)
......@@ -61,7 +61,7 @@ class ZoeScheduler:
def execution_terminate(self, execution: Execution) -> None:
"""
Inform the scheduler that an execution has been terminated.
Inform the master that an execution has been terminated.
:param execution: the terminated execution
:return: None
"""
......@@ -70,7 +70,7 @@ class ZoeScheduler:
def _check_runnable(self): # called periodically, does not use state
"""
This method is called by the main scheduler loop to check if there is an execution that can be run. In case there is
This method is called by the main master loop to check if there is an execution that can be run. In case there is
it starts it.
:return: None
"""
......
......@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.scheduler_policies.fifo import FIFOSchedulerPolicy
from zoe_master.scheduler_policies.fifo import FIFOSchedulerPolicy
......@@ -13,9 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.state.application import Application
from zoe_scheduler.state.execution import Execution
from zoe_scheduler.stats import SchedulerStats
from zoe_master.state.application import Application
from zoe_master.state.execution import Execution
from zoe_master.stats import SchedulerStats
class BaseSchedulerPolicy:
......
......@@ -15,11 +15,11 @@
import time
from zoe_scheduler.stats import SchedulerStats
from zoe_scheduler.scheduler_policies.base import BaseSchedulerPolicy
from zoe_scheduler.state.application import Application
from zoe_scheduler.state.execution import Execution
from zoe_scheduler.config import singletons
from zoe_master.stats import SchedulerStats
from zoe_master.scheduler_policies.base import BaseSchedulerPolicy
from zoe_master.state.application import Application
from zoe_master.state.execution import Execution
from zoe_master.config import singletons
class FIFOSchedulerPolicy(BaseSchedulerPolicy):
......
......@@ -16,8 +16,8 @@
import logging
import os
from zoe_scheduler.state.blobs import BaseBlobs
from zoe_scheduler.config import get_conf
from zoe_master.state.blobs import BaseBlobs
from zoe_master.config import get_conf
BLOB_PATH = 'blobs'
......
......@@ -14,8 +14,8 @@
# limitations under the License.
from zoe_lib.exceptions import ZoeException
from zoe_scheduler.state.base import BaseState
from zoe_scheduler.state.execution import Execution
from zoe_master.state.base import BaseState
from zoe_master.state.execution import Execution
class Container(BaseState):
......
......@@ -19,8 +19,8 @@ from io import BytesIO
import dateutil.parser
from zoe_scheduler.state.base import BaseState
from zoe_scheduler.state.application import Application
from zoe_master.state.base import BaseState
from zoe_master.state.application import Application
from zoe_lib.exceptions import ZoeException
......
......@@ -17,11 +17,11 @@ import logging
import json
from zoe_lib.exceptions import ZoeException
from zoe_scheduler.config import get_conf
from zoe_scheduler.state.user import User
from zoe_scheduler.state.execution import Execution
from zoe_scheduler.state.container import Container
from zoe_scheduler.state.blobs import BaseBlobs
from zoe_master.config import get_conf
from zoe_master.state.user import User
from zoe_master.state.execution import Execution
from zoe_master.state.container import Container
from zoe_master.state.blobs import BaseBlobs
log = logging.getLogger(__name__)
......
......@@ -15,8 +15,8 @@
from passlib.context import CryptContext
from zoe_scheduler.state.base import BaseState
from zoe_scheduler.config import get_conf
from zoe_master.state.base import BaseState
from zoe_master.config import get_conf
class User(BaseState):
......
......@@ -18,7 +18,7 @@ import time
import logging
from zoe_lib.swarm_client import SwarmClient
from zoe_scheduler.config import get_conf
from zoe_master.config import get_conf
log = logging.getLogger(__name__)
......
......@@ -16,9 +16,9 @@
import pytest
import json
from zoe_scheduler.config import load_configuration, get_conf
from zoe_scheduler.state.manager import StateManager
from zoe_scheduler.state.blobs.fs import FSBlobs
from zoe_master.config import load_configuration, get_conf
from zoe_master.state.manager import StateManager
from zoe_master.state.blobs.fs import FSBlobs
class TestConf:
......@@ -40,7 +40,7 @@ def configuration(request):
@pytest.fixture(scope='session')
def application_dict():
jsondata = open("zoe_scheduler/tests/resources/spark-wordcount-test.json", "r")
jsondata = open("zoe_master/tests/resources/spark-wordcount-test.json", "r")
return json.load(jsondata)
......
......@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.state.application import Application, Process, ProcessEndpoint
from zoe_master.state.application import Application, Process, ProcessEndpoint
def test_application():
......
......@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.state.container import Container
from zoe_master.state.container import Container
def test_execution(state_manager):
......
......@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.state.execution import Execution
from zoe_master.state.execution import Execution
def test_execution(state_manager):
......
......@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from zoe_scheduler.state.user import User
from zoe_master.state.user import User
def test_execution(state_manager):
......
......@@ -33,7 +33,7 @@ def load_configuration(test_conf=None):
args_for_writing_out_config_file=["--write-config"])
argparser.add_argument('--debug', action='store_true', help='Enable debug output')
argparser.add_argument('--swarm', help='Swarm/Docker API endpoint (ex.: zk://zk1:2181,zk2:2181 or http://swarm:2380)', default='http://localhost:2375')
argparser.add_argument('--scheduler-url', help='URL of the scheduler\'s REST API', default='http://127.0.0.1:4850')
argparser.add_argument('--master-url', help='URL of the master\'s REST API', default='http://127.0.0.1:4850')
argparser.add_argument('--zoeadmin-password', help='Password used to login as the master Zoe administrator', default='changeme')
argparser.add_argument('--container-name-prefix', help='String prefixed to all container names generated by this instance of Zoe', default='prod')
argparser.add_argument('--spark-activity-timeout', help='Number of seconds of inactivity (no jobs run) before a Spark cluster is terminated', type=int, default=18000)
......
......@@ -13,9 +13,9 @@ log = logging.getLogger(__name__)
def check_guests(swarm):
query_api = ZoeQueryAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
cont_api = ZoeContainerAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
query_api = ZoeQueryAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
exec_api = ZoeExecutionsAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
cont_api = ZoeContainerAPI(get_conf().master_url, 'zoeadmin', get_conf().zoeadmin_password)
guests = query_api.query('user', role='guest')
execs = exec_api.list()
......
......@@ -29,7 +29,7 @@ def main_callback(event):
def container_died(zoe_id: int):
log.debug('A container died')
# tell the scheduler via the rest api
# tell the master via the rest api
cont_api = ZoeContainerAPI(get_conf().scheduler_url, 'zoeadmin', get_conf().zoeadmin_password)
try:
cont_api.died(zoe_id)
......
......@@ -33,7 +33,7 @@ def load_configuration():
argparser.add_argument('--debug', action='store_true', help='Enable debug output')
argparser.add_argument('--listen-address', type=str, help='Address to listen to for incoming connections', default="0.0.0.0")
argparser.add_argument('--listen-port', type=int, help='Port to listen to for incoming connections', default=5001)
argparser.add_argument('--zoe-url', help='URL of the Zoe scheduler process', default='http://127.0.0.1:4850')
argparser.add_argument('--zoe-url', help='URL of the Zoe master process', default='http://127.0.0.1:4850')
argparser.add_argument('--zoe-admin-user', help='Admin-level user in Zoe', default='zoeadmin')
argparser.add_argument('--zoe-admin-pass', help='Password for the admin-level user in Zoe', default='changeme')
argparser.add_argument('--cookie-secret', help='key used to encrypt cookies', default="hr4h3H'kmn F8fz/;CJN5a!")
......
......@@ -19,7 +19,7 @@ import logging
from jinja2 import Template
from zoe_scheduler.state.execution import ExecutionState
from zoe_master.state.execution import ExecutionState
from common.configuration import zoe_conf
log = logging.getLogger(__name__)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment