Commit 5aa53f36 authored by Daniele Venzano's avatar Daniele Venzano

Tensorflow ZApp

parent ce35f7ad
# Created by .ignore support plugin (hsz.mobi)
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
.idea/
state.zoe
/zoe*.conf
zoepass.csv
stages:
- deploy
variables:
VERSION: $CI_BUILD_ID
images:
image: docker:latest
stage: deploy
before_script:
- mkdir -p /etc/docker/certs.d/$DOCKER_REGISTRY
- cp /registry-ca.crt /etc/docker/certs.d/$DOCKER_REGISTRY/ca.crt
- mkdir -p $HOME/.docker
- echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
- apk update
- apk add python findutils bash
script:
- bash build_all.sh
- python gen_json.py
artifacts:
paths:
- "*.json"
only:
- master
# TensorFlow image
This is a preliminary image for TensorFlow. It is used by Zoe, the Container Analytics as a
Service system to create a very simple TensorFlow batch service.
Zoe can be found at: https://github.com/DistributedSystemsGroup/zoe
## Setup
The Dockerfile runs a start script that:
* Launches your TensorFlow program until termination
* Launches a TensorBoard instance to inspect the output of your program
#!/usr/bin/env bash
set -e
if [ ! -d docker ]; then
exit
fi
REPOSITORY=${REPOSITORY:-zapps}
DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker-registry:5000}
VERSION=${VERSION:-`date +%Y%m%d%H%M%S`}
built_images=''
for d in `find docker -mindepth 1 -maxdepth 1 -type d -printf '%f '`; do
pushd docker/${d}
docker build -t ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION} .
docker push ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}
popd
built_images+="${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}\n"
done
echo "-------------END SCRIPT-----------------"
echo "Images built:"
printf ${built_images}
echo
FROM ubuntu:14.04
MAINTAINER Pietro Michiardi <Pietro.Michiardi@eurecom.fr>
ENV DEBIAN_FRONTEND noninteractive
##################################################
# BASELINE
##################################################
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends \
python-pip \
python-dev \
software-properties-common \
curl \
pkg-config \
zip \
g++ \
zlib1g-dev \
unzip \
swig \
git \
swig \
&& apt-get clean
RUN python -m pip install --upgrade pip
##################################################
# JAVA
##################################################
RUN add-apt-repository ppa:webupd8team/java -y
RUN /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
RUN apt-get update && apt-get install -y oracle-java8-installer
##################################################
# TENSORFLOW
##################################################
# From sources
# INSTALL BAZEL
RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
RUN apt-get update && sudo apt-get install -y --force-yes --no-install-recommends bazel patch
RUN apt-get upgrade -y --force-yes --no-install-recommends bazel
# CLONE FROM GIT
RUN git clone https://github.com/tensorflow/tensorflow
WORKDIR /tensorflow
##################################################
# Make sure important packages are installed
##################################################
RUN pip install --upgrade numpy pandas matplotlib
# COMPILE AND INSTALL PIP PACKAGE
COPY files/conf.txt /tmp/conf.txt
RUN ./configure < /tmp/conf.txt
RUN bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
RUN pip install wheel
RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
RUN pip install /tmp/tensorflow_pkg/tensorflow-*.whl
##################################################
# GPy
##################################################
RUN pip install --upgrade GPy paramz
##################################################
# GPFlow
##################################################
WORKDIR /opt
RUN git clone https://github.com/GPflow/GPflow.git
WORKDIR /opt/GPflow
RUN python setup.py install
##################################################
# SETUP ZOE APP
##################################################
# Configure environment
ENV JAVA_HOME /usr/lib/jvm/java-8-oracle/
ENV SHELL /bin/bash
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
# Configure container startup as root
EXPOSE 6006
WORKDIR /mnt/workspace
ENV WS_DIR /mnt/workspace
#COPY files/start-tf.sh /usr/local/bin/
#CMD ["/usr/local/bin/start-tf.sh"]
#RUN chmod 755 /usr/local/bin/start-tf.sh
#ADD files/* /usr/local/bin/
#RUN chmod +x /usr/local/bin/*
#ENTRYPOINT ["/usr/local/bin/start-tf.sh"]
#!/usr/bin/env bash
set -e
cd $ZOE_WORKSPACE
python "$@"
# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
import os
APP_NAME = 'tensorflow'
ZOE_APPLICATION_DESCRIPTION_VERSION = 3
options = {
'core_limit': {
'value': 4,
'description': 'Core limit'
},
'memory_limit': {
'value': 4 * (1024**3),
'description': 'Memory limit (bytes)'
}
}
REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
REPOSITORY = os.getenv("REPOSITORY", default="zapps")
VERSION = os.getenv("VERSION", default="latest")
GOOG_IMAGE = "gcr.io/tensorflow/tensorflow"
CUSTOM_IMAGE = REGISTRY + '/' + REPOSITORY + '/tensorflow:' + VERSION
def goog_tensorflow_service(memory_limit, core_limit):
"""
:rtype: dict
"""
service = {
'name': "jupyter",
'image': GOOG_IMAGE,
'monitor': True,
'resources': {
"memory": {
"min": memory_limit,
"max": memory_limit
},
"cores": {
"min": core_limit,
"max": core_limit
}
},
'ports': [
{
'name': "Tensorboard web interface",
'protocol': "tcp",
'port_number': 6006,
'url_template': "http://{ip_port}/"
},
{
'name': "Notebook web interface",
'protocol': "tcp",
'port_number': 8888,
'url_template': "http://{ip_port}/"
}
],
'environment': [],
'volumes': [],
'command': None,
'total_count': 1,
'essential_count': 1,
'startup_order': 0
}
return service
def custom_tensorflow_service(memory_limit, core_limit):
"""
:rtype: dict
"""
service = {
'name': "jupyter",
'image': CUSTOM_IMAGE,
'monitor': True,
'resources': {
"memory": {
"min": memory_limit,
"max": memory_limit
},
"cores": {
"min": core_limit,
"max": core_limit
}
},
'ports': [],
'environment': [],
'volumes': [],
'command': None,
'total_count': 1,
'essential_count': 1,
'startup_order': 0
}
return service
if __name__ == '__main__':
app = {
'name': APP_NAME,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
goog_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
]
}
json.dump(app, open("goog_tensorflow.json", "w"), sort_keys=True, indent=4)
app = {
'name': APP_NAME,
'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
'will_end': False,
'size': 512,
'services': [
custom_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
]
}
json.dump(app, open("custom_tensorflow.json", "w"), sort_keys=True, indent=4)
print("ZApp written")
logo.png

15.6 KB

Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment