Commit 4cd71c34 authored by Daniele Venzano's avatar Daniele Venzano

Update image and gitlab CI

parent 58028eb2
Pipeline #4776 failed with stages
in 14 seconds
......@@ -19,9 +19,6 @@ build:json:
build:hub:
stage: build
script:
- docker pull tensorflow/magenta
- docker pull tensorflow/syntaxnet
- docker pull gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3
- docker pull gcr.io/tensorflow/tensorflow:1.3.0-py3
test:json:
......@@ -30,14 +27,11 @@ test:json:
before_script:
- pip install requests
script:
- python /scripts/validate.py ${ZOE_VALIDATION_URL} *google.json
- python /scripts/validate.py ${ZOE_VALIDATION_URL} tf-google.json
pull:all:
stage: pull
script:
- /scripts/distribute_docker_image.sh tensorflow/magenta
- /scripts/distribute_docker_image.sh tensorflow/syntaxnet
- /scripts/distribute_docker_image.sh gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3
- /scripts/distribute_docker_image.sh gcr.io/tensorflow/tensorflow:1.3.0-py3
deploy:json:
......
# Batch TensorFlow
Stand-alone batch TensorFlow version 1.3 for Python 3, recompiled with instructions for modern x86_64 Intel CPUs.
This ZApp expects as parameter the command to run, relative to the workspace. For example, if you want to run
the script `~/algo/run.py`, you can write `python3 ./algo/run.py`.
# TensorFlow image
This is a preliminary image for TensorFlow. It is used by Zoe, the Container Analytics as a
Service system to create a very simple TensorFlow batch service.
Zoe can be found at: https://github.com/DistributedSystemsGroup/zoe
## Setup
The Dockerfile runs a start script that:
* Launches your TensorFlow program until termination
* Launches a TensorBoard instance to inspect the output of your program
# Tensorflow Magenta
Unmodified [TensorFlow Magenta](https://magenta.tensorflow.org) as generated by Google.
Please note that you need to get the Notebook key from the service logs to be able to access the Notebook.
# SyntaxNet TensorFlow
URL: [https://github.com/tensorflow/models/tree/master/syntaxnet](https://github.com/tensorflow/models/tree/master/syntaxnet)
DRAGNN (Dynamic Recurrent Acyclic Graphical Neural Networks) is a framework for building multi-task, dynamically connected computation graphs.
In practice, it uses SyntaxNet as its backend to perform several NLP tasks in a joint model (publicly released models include segmentation + POS tagging + syntactic parsing).
Please note that you need to get the Notebook key from the service logs to be able to access the Notebook.
#!/usr/bin/env bash
set -e
if [ ! -d docker ]; then
exit
fi
REPOSITORY=${REPOSITORY:-zapps}
DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker-registry:5000}
VERSION=${VERSION:-`date +%Y%m%d%H%M%S`}
built_images=''
for d in `find docker -mindepth 1 -maxdepth 1 -type d -printf '%f '`; do
pushd docker/${d}
docker build -t ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION} .
docker push ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}
popd
built_images+="${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}\n"
done
echo "-------------END SCRIPT-----------------"
echo "Images built:"
printf ${built_images}
echo
FROM ubuntu:14.04
MAINTAINER Pietro Michiardi <Pietro.Michiardi@eurecom.fr>
ENV DEBIAN_FRONTEND noninteractive
##################################################
# BASELINE
##################################################
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends \
python3-pip \
python3-dev \
software-properties-common \
curl \
pkg-config \
zip \
g++ \
zlib1g-dev \
unzip \
swig \
git \
swig \
&& apt-get clean
RUN python3 -m pip install --upgrade pip
##################################################
# JAVA
##################################################
RUN add-apt-repository ppa:webupd8team/java -y
RUN /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
RUN apt-get update && apt-get install -y oracle-java8-installer
##################################################
# TENSORFLOW
##################################################
# From sources
# INSTALL BAZEL
RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
RUN apt-get update && sudo apt-get install -y --force-yes --no-install-recommends bazel patch
RUN apt-get upgrade -y --force-yes --no-install-recommends bazel
##################################################
# Make sure important packages are installed
##################################################
RUN pip3 install --upgrade numpy pandas matplotlib
# Running bazel inside a `docker build` command causes trouble, cf:
# https://github.com/bazelbuild/bazel/issues/134
# The easiest solution is to set up a bazelrc file forcing --batch.
RUN echo "startup --batch" >>/etc/bazel.bazelrc
# Similarly, we need to workaround sandboxing issues:
# https://github.com/bazelbuild/bazel/issues/418
RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
>>/etc/bazel.bazelrc
# Install the most recent bazel release.
ENV BAZEL_VERSION 0.5.0
WORKDIR /
RUN mkdir /bazel && \
cd /bazel && \
curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
chmod +x bazel-*.sh && \
./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
cd / && \
rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
# Download and build TensorFlow.
RUN git clone https://github.com/tensorflow/tensorflow.git && \
cd tensorflow && \
git checkout r1.3
WORKDIR /tensorflow
# TODO(craigcitro): Don't install the pip package, since it makes it
# more difficult to experiment with local changes. Instead, just add
# the built directory to the path.
ENV CI_BUILD_PYTHON python3
RUN pip3 --no-cache-dir install wheel
RUN sed -i '\@https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz@d' tensorflow/workspace.bzl
RUN tensorflow/tools/ci_build/builds/configured CPU \
bazel build -c opt --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
tensorflow/tools/pip_package:build_pip_package && \
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
pip3 --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
rm -rf /tmp/pip && \
rm -rf /root/.cache
# Clean up pip wheel and Bazel cache when done.
# COMPILE AND INSTALL PIP PACKAGE
#COPY files/conf.txt /tmp/conf.txt
#RUN ./configure < /tmp/conf.txt
#RUN bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
#RUN pip install wheel
#RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
#RUN pip install /tmp/tensorflow_pkg/tensorflow-*.whl
##################################################
# GPy
##################################################
RUN pip3 install --upgrade GPy paramz
##################################################
# GPFlow
##################################################
WORKDIR /opt
RUN git clone https://github.com/GPflow/GPflow.git
WORKDIR /opt/GPflow
RUN python3 setup.py install
##################################################
# SETUP ZOE APP
##################################################
# Configure environment
ENV JAVA_HOME /usr/lib/jvm/java-8-oracle/
ENV SHELL /bin/bash
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
# Configure container startup as root
EXPOSE 6006
WORKDIR /mnt/workspace
ENV WS_DIR /mnt/workspace
#COPY files/start-tf.sh /usr/local/bin/
#CMD ["/usr/local/bin/start-tf.sh"]
#RUN chmod 755 /usr/local/bin/start-tf.sh
#ADD files/* /usr/local/bin/
#RUN chmod +x /usr/local/bin/*
#ENTRYPOINT ["/usr/local/bin/start-tf.sh"]
#!/usr/bin/env bash
set -e
cd $ZOE_WORKSPACE
python "$@"
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
import os
APPS = [
('tf-google', "gcr.io/tensorflow/tensorflow:1.3.0-py3", False),
('stnet-google', "tensorflow/syntaxnet", False),
('mag-google', "tensorflow/magenta", False),
('tf-google-gpu', "gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3", True)
]
ZOE_APPLICATION_DESCRIPTION_VERSION = 3
options = {
'core_limit': {
'value': 4,
'description': 'Core limit'
},
'memory_limit': {
'value': 32 * (1024**3),
'description': 'Memory limit (bytes)'
}
}
REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
REPOSITORY = os.getenv("REPOSITORY", default="zapps")
VERSION = os.getenv("VERSION", default="latest")
def goog_tensorflow_service(memory_limit, core_limit, image, gpu=False):
"""
:rtype: dict
"""
service = {
'name': "tf-jupyter",
'image': image,
'monitor': True,
'resources': {
"memory": {
"min": memory_limit,
"max": memory_limit
},
"cores": {
"min": core_limit,
"max": core_limit
}
},
'ports': [
{
'name': "Tensorboard web interface",
'protocol': "tcp",
'port_number': 6006,
'url_template': "http://{ip_port}/"
},
{
'name': "Notebook web interface",
'protocol': "tcp",
'port_number': 8888,
'url_template': "http://{ip_port}/"
}
],
'environment': [],
'volumes': [],
'command': None,
'total_count': 1,
'essential_count': 1,
'startup_order': 0,
'replicas': 1
}
if gpu:
service['labels'] = ['gpu']
service['environment'] = [['NVIDIA_VISIBLE_DEVICES', 'all']]
return service
if __name__ == '__main__':
for app_name, image, gpu in APPS:
app = {
'name': app_name,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
goog_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"], image, gpu)
]
}
json.dump(app, open(app_name + ".json", "w"), sort_keys=True, indent=4)
print("ZApps written")
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
import os
APP_NAME = 'tf-standalone'
ZOE_APPLICATION_DESCRIPTION_VERSION = 3
options = {
'core_limit': {
'value': 4,
'description': 'Core limit'
},
'memory_limit': {
'value': 4 * (1024**3),
'description': 'Memory limit (bytes)'
}
}
REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
REPOSITORY = os.getenv("REPOSITORY", default="zapps")
VERSION = os.getenv("VERSION", default="latest")
CUSTOM_IMAGE = REGISTRY + '/' + REPOSITORY + '/tensorflow:' + VERSION
def custom_tensorflow_service(memory_limit, core_limit):
"""
:rtype: dict
"""
service = {
'name': "tf",
'image': CUSTOM_IMAGE,
'monitor': True,
'resources': {
"memory": {
"min": memory_limit,
"max": memory_limit
},
"cores": {
"min": core_limit,
"max": core_limit
}
},
'ports': [],
'environment': [],
'volumes': [],
'command': None,
'total_count': 1,
'essential_count': 1,
'startup_order': 0,
'replicas': 1
}
return service
if __name__ == '__main__':
app = {
'name': APP_NAME,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
custom_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
]
}
json.dump(app, open("custom_tensorflow.json", "w"), sort_keys=True, indent=4)
print("ZApp written")
......@@ -8,13 +8,6 @@
"readable_descr": "README-goog.md",
"parameters": []
},
{
"category": "Machine learning",
"name": "Google TensorFlow notebook with GPU",
"description": "tf-google-gpu.json",
"readable_descr": "README-goog.md",
"parameters": []
},
{
"category": "Non-interactive",
"name": "Google TensorFlow batch",
......@@ -30,20 +23,6 @@
"default": "./my-tf-app/main.py"
}
]
},
{
"category": "Tutorials and examples",
"name": "DRAGNN SyntaxNet model",
"description": "stnet-google.json",
"readable_descr": "README-syntaxnet.md",
"parameters": []
},
{
"category": "Tutorials and examples",
"name": "Magenta model",
"description": "mag-google.json",
"readable_descr": "README-magenta.md",
"parameters": []
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment