Commit 2bdff1e8 authored by Daniele Venzano's avatar Daniele Venzano

Use standard images from Jupyter project

parent f80930af
stages:
- build
- test
- pull
- deploy
variables:
VERSION: $CI_BUILD_ID
VERSION: $CI_PIPELINE_ID
images:
image: docker:latest
stage: deploy
before_script:
- mkdir -p /etc/docker/certs.d/$DOCKER_REGISTRY
- cp /registry-ca.crt /etc/docker/certs.d/$DOCKER_REGISTRY/ca.crt
- mkdir -p $HOME/.docker
- echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
- apk update
- apk add python findutils bash
build:json:
image: python:3-alpine
stage: build
script:
- bash build_all.sh
- python gen_json.py
artifacts:
paths:
- "*.json"
test:json:
image: python:3-alpine
stage: test
before_script:
- pip install requests
script:
- python validate.py ${ZOE_VALIDATION_URL} *notebook.json
pull:all:
stage: pull
script:
- docker -H ${SWARM} ${SWARM_TLS_OPTIONS} pull jupyter/datascience-notebook
- docker -H ${SWARM} ${SWARM_TLS_OPTIONS} pull jupyter/r-notebook
- docker -H ${SWARM} ${SWARM_TLS_OPTIONS} pull jupyter/scipy-notebook
- docker -H ${SWARM} ${SWARM_TLS_OPTIONS} pull jupyter/tensorflow-notebook
only:
- master
deploy:json:
stage: deploy
script:
- mkdir -p ${ZAPP_SHOP_BASE_PATH}/${CI_PROJECT_NAME}
- cp *.json ${ZAPP_SHOP_BASE_PATH}/${CI_PROJECT_NAME}/
- cp logo.png ${ZAPP_SHOP_BASE_PATH}/${CI_PROJECT_NAME}/
- cp README*.md ${ZAPP_SHOP_BASE_PATH}/${CI_PROJECT_NAME}/
artifacts:
paths:
- "*.json"
- logo.png
- "README*.md"
only:
- master
# Jupyter Notebook Data Science Stack
URL: [https://hub.docker.com/r/jupyter/datascience-notebook/](https://hub.docker.com/r/jupyter/datascience-notebook/)
* Jupyter Notebook 5.0.x
* Conda Python 3.x environment
* pandas, matplotlib, scipy, seaborn, scikit-learn, scikit-image, sympy, cython, patsy, statsmodel, cloudpickle, dill, numba, bokeh pre-installed
* Conda R v3.3.x and channel
* plyr, devtools, shiny, rmarkdown, forecast, rsqlite, reshape2, nycflights13, caret, rcurl, and randomforest pre-installed
* The tidyverse R packages are also installed, including ggplot2, dplyr, tidyr, readr, purrr, tibble, stringr, lubridate, and broom
* Julia v0.5.x with Gadfly, RDatasets and HDF5 pre-installed
# Jupyter Notebook R Stack
URL: [https://hub.docker.com/r/jupyter/r-notebook/](https://hub.docker.com/r/jupyter/r-notebook/)
* Jupyter Notebook 5.0.x
* Conda R v3.3.x and channel
* plyr, devtools, shiny, rmarkdown, forecast, rsqlite, reshape2, nycflights13, caret, rcurl, and randomforest pre-installed
* The tidyverse R packages are also installed, including ggplot2, dplyr, tidyr, readr, purrr, tibble, stringr, lubridate, and broom
# Jupyter Notebook Scientific Python Stack
URL: [https://hub.docker.com/r/jupyter/scipy-notebook/](https://hub.docker.com/r/jupyter/scipy-notebook/)
* Jupyter Notebook 5.0.x
* Conda Python 3.x environment
* pandas, matplotlib, scipy, seaborn, scikit-learn, scikit-image, sympy, cython, patsy, statsmodel, cloudpickle, dill, numba, bokeh, vincent, beautifulsoup, xlrd pre-installed
# Jupyter Notebook Scientific Python Stack + Tensorflow
Maintainer: Daniele Venzano <daniele.venzano@eurecom.fr>
URL: [https://hub.docker.com/r/jupyter/tensorflow-notebook/](https://hub.docker.com/r/jupyter/tensorflow-notebook/)
* Everything in [Scipy](https://github.com/jupyter/docker-stacks/tree/master/scipy-notebook) Notebook
* Tensorflow and Keras for Python 3.x (without GPU support)
# Jupyter ZApp
Maintainer: Daniele Venzano <daniele.venzano@eurecom.fr>
URL: [http://jupyter.org/](http://jupyter.org/)
Starts a Jupyter Notebook with Python, R and Scala kernels
FROM ubuntu:14.04
MAINTAINER Daniele Venzano <venza@brownhat.org>
RUN apt-get update && apt-get install -y --force-yes software-properties-common python-software-properties
RUN apt-add-repository -y ppa:webupd8team/java
RUN /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
RUN apt-get update && apt-get -y install oracle-java7-installer oracle-java7-set-default curl
ENV JAVA_HOME /usr/lib/jvm/java-7-oracle/
RUN sudo sh -c 'echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list'
RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com E084DAB9
WORKDIR /opt
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends \
git \
vim \
wget \
build-essential \
python-dev \
ca-certificates \
bzip2 \
unzip \
libsm6 \
pandoc \
texlive-latex-base \
texlive-latex-extra \
texlive-fonts-extra \
texlive-fonts-recommended \
texlive-generic-recommended \
sudo \
locales \
libxrender1 \
libopenblas-dev \
r-base \
libcurl4-openssl-dev \
libssl-dev \
&& apt-get clean
RUN locale-gen en_US.UTF-8
# Install Tini
RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.6.0/tini && \
echo "d5ed732199c36a1189320e6c4859f0169e950692f451c03e7854243b95f4234b *tini" | sha256sum -c - && \
mv tini /usr/local/bin/tini && \
chmod +x /usr/local/bin/tini
# Configure environment
ENV CONDA_DIR /opt/conda
ENV PATH $CONDA_DIR/bin:$PATH
ENV SHELL /bin/bash
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
RUN mkdir /root/work && \
mkdir /root/.jupyter && \
mkdir /root/.local
RUN cd /tmp && \
mkdir -p $CONDA_DIR && \
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
/bin/bash Miniconda3-latest-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
rm Miniconda3-latest-Linux-x86_64.sh && \
$CONDA_DIR/bin/conda install --yes conda
RUN conda install --yes \
'notebook' \
'terminado' \
'ipywidgets' \
'pandas' \
'matplotlib' \
'scipy' \
'seaborn' \
'scikit-learn' \
'statsmodels' \
'ipyparallel' \
'basemap' \
'pillow' \
&& conda clean -yt
RUN /opt/conda/bin/pip install thunder-python showit haversine
RUN wget https://oss.sonatype.org/content/repositories/snapshots/com/github/alexarchambault/jupyter/jupyter-scala-cli_2.11.6/0.2.0-SNAPSHOT/jupyter-scala_2.11.6-0.2.0-SNAPSHOT.tar.xz
RUN tar xvfJ jupyter-scala_2.11.6-0.2.0-SNAPSHOT.tar.xz && rm jupyter-scala_2.11.6-0.2.0-SNAPSHOT.tar.xz
RUN jupyter-scala_2.11.6-0.2.0-SNAPSHOT/bin/jupyter-scala
RUN R -e "install.packages(c('pbdZMQ', 'repr', 'devtools'), repos = 'http://cran.rstudio.com/')"
RUN R -e "devtools::install_github('irkernel/IRdisplay')"
RUN R -e "devtools::install_github('irkernel/IRkernel')"
RUN R -e "IRkernel::installspec(user = FALSE)"
# Configure container startup as root
EXPOSE 8888
WORKDIR /mnt/workspace
ENTRYPOINT ["tini", "--"]
CMD ["start-notebook.sh"]
# Add local files as late as possible to avoid cache busting
COPY files/start-notebook.sh /usr/local/bin/
RUN chmod 755 /usr/local/bin/start-notebook.sh
COPY files/jupyter_notebook_config.py /root/.jupyter/
RUN mkdir -p /root/.ipython/profile_default/startup/
# Jupyter Notebook image
This image contains the Jupyter notebook with a number of useful kernels. It is used by Zoe, the Container Analytics as a
Service system to create on-demand notebooks connected to containerized Spark clusters.
Zoe can be found at: https://github.com/DistributedSystemsGroup/zoe
## Setup
The Dockerfile runs a start script that configures the Notebook using these environment variables:
* SPARK\_MASTER\_IP: IP address of the Spark master this notebook should use for its kernel
* PROXY\_ID: string to use as a prefix for URL paths, for reverse proxying
* SPARK\_EXECUTOR\_RAM: How much RAM to use for each executor spawned by the notebook
# Copyright (c) Jupyter Development Team.
from jupyter_core.paths import jupyter_data_dir
import subprocess
import os
import errno
import stat
PEM_FILE = os.path.join(jupyter_data_dir(), 'notebook.pem')
c = get_config()
c.NotebookApp.ip = '*'
c.NotebookApp.port = 8888
c.NotebookApp.open_browser = False
# Set a certificate if USE_HTTPS is set to any value
if 'USE_HTTPS' in os.environ:
if not os.path.isfile(PEM_FILE):
# Ensure PEM_FILE directory exists
dir_name = os.path.dirname(PEM_FILE)
try:
os.makedirs(dir_name)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(dir_name):
pass
else: raise
# Generate a certificate if one doesn't exist on disk
subprocess.check_call(['openssl', 'req', '-new',
'-newkey', 'rsa:2048', '-days', '365', '-nodes', '-x509',
'-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
'-keyout', PEM_FILE, '-out', PEM_FILE])
# Restrict access to PEM_FILE
os.chmod(PEM_FILE, stat.S_IRUSR | stat.S_IWUSR)
c.NotebookApp.certfile = PEM_FILE
# Set a password if PASSWORD is set
if 'PASSWORD' in os.environ:
from IPython.lib import passwd
c.NotebookApp.password = passwd(os.environ['PASSWORD'])
del os.environ['PASSWORD']
#!/usr/bin/env bash
set -x
R -e "IRkernel::installspec()"
ipcluster nbextension enable
rm -f jupyter_notebook_config.json
exec jupyter notebook --NotebookApp.token='' --allow-root $*
......@@ -13,13 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Spark-Jupyter Zoe application description generator."""
"""Jupyter Zoe application description generator."""
import json
import sys
import os
APP_NAME = 'jupyter-notebook'
UPSTREAM_NOTEBOOKS = [
("r-notebook", "jupyter/r-notebook"),
("datascience-notebook", "jupyter/datascience-notebook"),
("tensorflow-notebook", "jupyter/tensorflow-notebook"),
("scipy-notebook", "jupyter/scipy-notebook")
]
ZOE_APPLICATION_DESCRIPTION_VERSION = 3
options = {
......@@ -37,15 +43,13 @@ REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
REPOSITORY = os.getenv("REPOSITORY", default="zapps")
VERSION = os.getenv("VERSION", default="latest")
IMAGE = REGISTRY + '/' + REPOSITORY + '/jupyter-notebook:' + VERSION
def jupyter_service(memory_limit, core_limit):
def jupyter_service(memory_limit, core_limit, image):
"""
:rtype: dict
"""
service = {
'name': "jupyter",
'image': IMAGE,
'image': image,
'monitor': True,
'resources': {
"memory": {
......@@ -65,9 +69,7 @@ def jupyter_service(memory_limit, core_limit):
'url_template': 'http://{ip_port}/'
}
],
'environment': [
["NB_USER", "{user_name}"]
],
'environment': [],
'volumes': [],
'command': None,
'total_count': 1,
......@@ -79,17 +81,18 @@ def jupyter_service(memory_limit, core_limit):
if __name__ == '__main__':
app = {
'name': APP_NAME,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
jupyter_service(options["memory_limit"]["value"], options["core_limit"]["value"])
]
}
for app_name, image in UPSTREAM_NOTEBOOKS:
app = {
'name': app_name,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
jupyter_service(options["memory_limit"]["value"], options["core_limit"]["value"], image)
]
}
json.dump(app, open("jupyter.json", "w"), sort_keys=True, indent=4)
json.dump(app, open(app_name + ".json", "w"), sort_keys=True, indent=4)
print("ZApp written")
print("ZApps written")
{
"version": 1,
"category": "Jupyter notebooks",
"zapps": [
{
"name": "Python, Scala and R notebook",
"description": "jupyter.json",
"category": "Jupyter notebooks",
"readable_descr": "README-datascience.md",
"name": "Data science notebook",
"description": "datascience-notebook.json",
"parameters": []
},
{
"category": "Jupyter notebooks",
"readable_descr": "README-r.md",
"name": "R notebook",
"description": "r-notebook.json",
"parameters": []
},
{
"category": "Jupyter notebooks",
"readable_descr": "README-scipy.md",
"name": "SciPy notebook",
"description": "scipy-notebook.json",
"parameters": []
},
{
"category": "Jupyter notebooks",
"readable_descr": "README-tensorflow.md",
"name": "TensorFlow notebook",
"description": "tensorflow-notebook.json",
"parameters": []
}
]
}
#!/usr/bin/python
import json
import sys
import requests
URL = sys.argv[1]
files = sys.argv[2:]
for f in files:
data = json.load(open(f, encoding="utf-8"))
data_req = { "application": data }
r = requests.post(URL, json=data_req)
if r.status_code != 200:
rep = r.json()
print(rep['message'])
sys.exit(1)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment