Commit 15a01506 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Fix bug, pass download URL to submit script in container

parent 91ffbae6
from common.state.execution import Execution
from common.state import Proxy, AlchemySession
from common.state import Proxy, AlchemySession, Application
from common.configuration import conf
......@@ -13,3 +13,7 @@ def generate_notebook_url(execution: Execution) -> str:
c = execution.find_container("spark-notebook")
pr = state.query(Proxy).filter_by(container_id=c.id, service_name="Spark Notebook interface").one()
return 'http://' + conf['web_server_name'] + conf['proxy_path_prefix'] + '/{}'.format(pr.id)
def generate_application_binary_url(application: Application) -> str:
return 'http://' + conf['web_server_name'] + '/api/applications/download/{}'.format(application.id)
......@@ -13,12 +13,12 @@ from common.exceptions import UserIDDoesNotExist, ApplicationStillRunning
import common.object_storage as storage
from common.configuration import conf
REGISTRY = "10.0.0.2:5000"
MASTER_IMAGE = REGISTRY + "/zoe/spark-master-1.4.1:1.2"
WORKER_IMAGE = REGISTRY + "/zoe/spark-worker-1.4.1:1.2"
SHELL_IMAGE = REGISTRY + "/zoe/spark-shell-1.4.1:1.2"
SUBMIT_IMAGE = REGISTRY + "/zoe/spark-submit-1.4.1:1.2"
NOTEBOOK_IMAGE = REGISTRY + "/zoe/spark-notebook-1.4.1:1.2"
REGISTRY = "10.1.0.1:5000"
MASTER_IMAGE = REGISTRY + "/zoe/spark-master-1.4.1:1.3"
WORKER_IMAGE = REGISTRY + "/zoe/spark-worker-1.4.1:1.3"
SHELL_IMAGE = REGISTRY + "/zoe/spark-shell-1.4.1:1.3"
SUBMIT_IMAGE = REGISTRY + "/zoe/spark-submit-1.4.1:1.3"
NOTEBOOK_IMAGE = REGISTRY + "/zoe/spark-notebook-1.4.1:1.3"
class ZoeClient:
......
......@@ -13,6 +13,7 @@ from common.application_resources import ApplicationResources
from common.exceptions import CannotCreateCluster
from common.configuration import conf
from common.object_storage import logs_archive_upload
from common.urls import generate_application_binary_url
log = logging.getLogger(__name__)
......@@ -143,7 +144,7 @@ class PlatformManager:
cli_opts.add_env_variable("PROXY_ID", container.id)
cli_opts.add_env_variable("APPLICATION_ID", application.id)
cli_opts.add_env_variable("SPARK_OPTIONS", execution.spark_opts)
cli_opts.add_env_variable("REDIS_CLI_OPTIONS", "-h {} -p {} -n {}".format(conf["redis_server"], conf["redis_port"], conf["redis_db"]))
cli_opts.add_env_variable("APPLICATION_URL", generate_application_binary_url(application))
cli_opts.add_env_variable("SPARK_OPTIONS", execution.spark_opts)
if "memory_limit" in execution.assigned_resources.worker_resources:
cli_opts.add_env_variable("SPARK_EXECUTOR_RAM", execution.assigned_resources.worker_resources["memory_limit"])
......
......@@ -84,6 +84,8 @@ class ZoeScheduler:
log.debug("Found a runnable execution!")
if self.platform.start_execution(execution_id, resources):
self.scheduler_policy.started(execution_id, resources)
else: # Some error happened
log.error('Execution ID {} cannot be started'.format(execution_id))
def schedule(self):
self._check_runnable()
......
class SparkClusterDescription:
def __init__(self):
self.num_workers = 2
self.executor_ram_size = "4g"
self.worker_cores = "2"
def for_spark_notebook(self):
self.num_workers = 2
self.worker_cores = "2"
self.executor_ram_size = "4g"
def for_spark_app(self, app_id):
self.num_workers = 4
self.worker_cores = "5"
self.executor_ram_size = "8g"
from zoe_web.sql import CAaaState
from zoe_web.config_parser import config
def _generate_proxied_url(proxy_id):
if proxy_id is not None:
return config.proxy_base_url + "/" + proxy_id
else:
return None
def get_container_addresses(container_id):
state = CAaaState()
proxy_list = state.get_proxies(container_id=container_id)
urls = []
for p in proxy_list:
external_url = _generate_proxied_url(p["id"])
urls.append((p["service_name"], external_url))
return urls
def get_notebook_address(cluster_id):
state = CAaaState()
proxy_id = state.get_proxy_for_service(cluster_id, "notebook")
return _generate_proxied_url(proxy_id)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment