Commit 75c92cf0 authored by Daniele Venzano's avatar Daniele Venzano

Update image and gitlab CI

parent 0e08fd06
Pipeline #4782 failed with stages
in 2 minutes and 33 seconds
......@@ -4,25 +4,10 @@ stages:
- push
- deploy
variables:
VERSION: $CI_PIPELINE_ID
REPOSITORY: zapps
before_script:
- mkdir -p $HOME/.docker
- echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
build:pytorch:
stage: build
script:
- docker build -t ${REPOSITORY}/pytorch:${VERSION} docker/pytorch
build:pull-hub:
stage: build
script:
- docker pull jupyter/datascience-notebook:latest
- docker pull jupyter/r-notebook:latest
- docker pull jupyter/tensorflow-notebook:latest
build:json:
image: python:3-alpine
......@@ -39,20 +24,14 @@ test:json:
before_script:
- pip install requests
script:
- python /scripts/validate.py ${ZOE_VALIDATION_URL} *notebook.json
- python /scripts/validate.py ${ZOE_VALIDATION_URL} datasci-notebook.json
push:all:
stage: push
script:
- /scripts/distribute_docker_image.sh jupyter/datascience-notebook:latest
- /scripts/distribute_docker_image.sh jupyter/r-notebook:latest
- /scripts/distribute_docker_image.sh jupyter/tensorflow-notebook:latest
- /scripts/distribute_docker_image.sh ${REPOSITORY}/pytorch:${VERSION}
push:hub:
stage: push
script:
- docker push ${REPOSITORY}/pytorch:${VERSION}
only:
- master
deploy:json:
stage: deploy
......
# Jupyter Notebook with PyTorch
URL: [https://hub.docker.com/r/jupyter/scipy-notebook/](https://hub.docker.com/r/jupyter/scipy-notebook/) and [http://pytorch.org/](http://pytorch.org/)
* Jupyter Notebook 5.0.x
* Conda Python 3.x environment
* pandas, matplotlib, scipy, seaborn, scikit-learn, scikit-image, sympy, cython, patsy, statsmodel, cloudpickle, dill, numba, bokeh, vincent, beautifulsoup, xlrd pre-installed
* PyTorch
Please note that you need to retrieve the secret key from the service logs to be able to access the notebooks.
# Jupyter Notebook R Stack
URL: [https://hub.docker.com/r/jupyter/r-notebook/](https://hub.docker.com/r/jupyter/r-notebook/)
* Jupyter Notebook 5.0.x
* Conda R v3.3.x and channel
* plyr, devtools, shiny, rmarkdown, forecast, rsqlite, reshape2, nycflights13, caret, rcurl, and randomforest pre-installed
* The tidyverse R packages are also installed, including ggplot2, dplyr, tidyr, readr, purrr, tibble, stringr, lubridate, and broom
Please note that you need to retrieve the secret key from the service logs to be able to access the notebooks.
# Jupyter Notebook Scientific Python Stack
URL: [https://hub.docker.com/r/jupyter/scipy-notebook/](https://hub.docker.com/r/jupyter/scipy-notebook/)
* Jupyter Notebook 5.0.x
* Conda Python 3.x environment
* pandas, matplotlib, scipy, seaborn, scikit-learn, scikit-image, sympy, cython, patsy, statsmodel, cloudpickle, dill, numba, bokeh, vincent, beautifulsoup, xlrd pre-installed
Please note that you need to retrieve the secret key from the service logs to be able to access the notebooks.
# Jupyter Notebook Scientific Python Stack + Tensorflow
Maintainer: Daniele Venzano <daniele.venzano@eurecom.fr>
URL: [https://hub.docker.com/r/jupyter/tensorflow-notebook/](https://hub.docker.com/r/jupyter/tensorflow-notebook/)
* Everything in [Scipy](https://github.com/jupyter/docker-stacks/tree/master/scipy-notebook) Notebook
* Tensorflow and Keras for Python 3.x (without GPU support)
Please note that you need to retrieve the secret key from the service logs to be able to access the notebooks.
#!/usr/bin/env bash
set -e
if [ ! -d docker ]; then
exit
fi
REPOSITORY=${REPOSITORY:-zapps}
DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker-registry:5000}
VERSION=${VERSION:-`date +%Y%m%d%H%M%S`}
built_images=''
for d in `find docker -mindepth 1 -maxdepth 1 -type d -printf '%f '`; do
pushd docker/${d}
docker build -t ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION} .
docker push ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}
popd
built_images+="${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}\n"
done
echo "-------------END SCRIPT-----------------"
echo "Images built:"
printf ${built_images}
echo
{
"name": "datasci-notebook",
"services": [
{
"command": null,
"environment": [
[
"NB_UID",
"1000"
],
[
"HOME",
"/mnt/workspace"
]
],
"essential_count": 1,
"image": "jupyter/datascience-notebook:latest",
"monitor": true,
"name": "jupyter",
"ports": [
{
"name": "Jupyter Notebook interface",
"port_number": 8888,
"protocol": "tcp",
"url_template": "http://{ip_port}/"
}
],
"replicas": 1,
"resources": {
"cores": {
"max": 4,
"min": 4
},
"memory": {
"max": 4294967296,
"min": 4294967296
}
},
"startup_order": 0,
"total_count": 1,
"volumes": [],
"work_dir": "/mnt/workspace"
}
],
"size": 512,
"version": 3,
"will_end": false
}
\ No newline at end of file
FROM jupyter/scipy-notebook:latest
MAINTAINER Daniele Venzano <venza@brownhat.org>
RUN conda install pytorch torchvision cuda80 -c soumith
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Jupyter Zoe application description generator."""
import json
import sys
import os
REPOSITORY = os.getenv("REPOSITORY", default="zapps")
VERSION = os.getenv("VERSION", default="latest")
UPSTREAM_NOTEBOOKS = [
("r-notebook", "jupyter/r-notebook:latest"),
("datasci-notebook", "jupyter/datascience-notebook:latest"),
("tf-notebook", "jupyter/tensorflow-notebook:latest"),
("pytorch-notebook", REPOSITORY + "/pytorch:" + VERSION)
]
ZOE_APPLICATION_DESCRIPTION_VERSION = 3
options = {
'core_limit': {
'value': 4,
'description': 'Notebook core limit'
},
'memory_limit': {
'value': 4 * (1024**3),
'description': 'Notebook memory limit (bytes)'
}
}
def jupyter_service(memory_limit, core_limit, image):
"""
:rtype: dict
"""
service = {
'name': "jupyter",
'image': image,
'monitor': True,
'resources': {
"memory": {
"min": memory_limit,
"max": memory_limit
},
"cores": {
"min": core_limit,
"max": core_limit
}
},
'ports': [
{
'name': 'Jupyter Notebook interface',
'protocol': 'tcp',
'port_number': 8888,
'url_template': 'http://{ip_port}/'
}
],
'environment': [
['NB_UID', '1000'],
['HOME', '/mnt/workspace']
],
'volumes': [],
'command': None,
'work_dir': '/mnt/workspace',
'total_count': 1,
'essential_count': 1,
'replicas': 1,
'startup_order': 0
}
return service
if __name__ == '__main__':
for app_name, image in UPSTREAM_NOTEBOOKS:
app = {
'name': app_name,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
jupyter_service(options["memory_limit"]["value"], options["core_limit"]["value"], image)
]
}
json.dump(app, open(app_name + ".json", "w"), sort_keys=True, indent=4)
print("ZApps written")
......@@ -7,35 +7,6 @@
"name": "Data science notebook",
"description": "datasci-notebook.json",
"parameters": []
},
{
"category": "Data science",
"readable_descr": "README-r.md",
"name": "R notebook",
"description": "r-notebook.json",
"parameters": []
},
{
"category": "Data science",
"readable_descr": "README-scipy.md",
"name": "SciPy notebook",
"description": "scipy-notebook.json",
"parameters": []
},
{
"category": "Machine learning",
"readable_descr": "README-tensorflow.md",
"name": "TensorFlow notebook",
"description": "tf-notebook.json",
"parameters": []
},
{
"category": "Machine learning",
"readable_descr": "README-pytorch.md",
"name": "PyTorch notebook",
"description": "pytorch-notebook.json",
"parameters": [],
"logo": "pytorch.png"
}
]
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment