Commit a8fccfd2 authored by Daniele Venzano's avatar Daniele Venzano
Browse files

Testing and fixing the Spark ZApp

parent 054094f4
Pipeline #10400 passed with stages
in 6 minutes and 49 seconds
......@@ -189,7 +189,7 @@ def spark_jupyter_notebook_service(mem_limit, worker_mem_limit):
'essential_count': 1,
'startup_order': 0,
'replicas': 1,
'command': 'jupyter lab --no-browser --NotebookApp.token=\'\' --allow-root --ip=0.0.0.0'
'command': None
}
return service
......
......@@ -103,11 +103,11 @@ COPY files/spark-defaults.conf /opt/spark-defaults.conf
# Configure container startup as root
EXPOSE 4040 8888
CMD /opt/conda/bin/jupyter lab --no-browser --NotebookApp.token='' --allow-root --ip=0.0.0.0
CMD ["/opt/start-notebook.sh"]
# Add local files as late as possible to avoid cache busting
COPY files/custom_startup.sh /opt
RUN chmod 755 /opt/custom_startup.sh
COPY files/*.sh /opt
RUN chmod 755 /opt/*.sh
COPY files/00-pyspark-setup.py /opt/
......
......@@ -21,5 +21,5 @@ COPY files/* /opt/
RUN chmod +x /opt/*.sh
EXPOSE 8080 7077
CMD ${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.master.Master --host ${SPARK_MASTER_IP} --port 7077 --webui-port 8080
CMD ["/opt/start_master.sh"]
......@@ -70,7 +70,4 @@ EXPOSE 8888 8081
ENV SPARK_WORKER_PORT 8888
ENV SPARK_WORKER_WEBUI_PORT 8081
CMD /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker \
spark://${SPARK_MASTER_IP}:7077 --cores ${SPARK_WORKER_CORES:-4} --memory ${SPARK_WORKER_RAM:-4g} \
-h ${SPARK_LOCAL_IP:-127.0.0.1}
CMD ["/opt/start-worker.sh"]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment