Tensorflow ZApp

.gitignore

+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+.idea/
+state.zoe
+/zoe*.conf
+zoepass.csv

.gitlab-ci.yml

+stages:
+  - deploy
+
+variables:
+  VERSION: $CI_BUILD_ID
+
+images:
+  image: docker:latest
+  stage: deploy
+  before_script:
+    - mkdir -p /etc/docker/certs.d/$DOCKER_REGISTRY
+    - cp /registry-ca.crt /etc/docker/certs.d/$DOCKER_REGISTRY/ca.crt
+    - mkdir -p $HOME/.docker
+    - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
+    - apk update
+    - apk add python findutils bash
+  script:
+    - bash build_all.sh
+    - python gen_json.py
+  artifacts:
+    paths:
+      - "*.json"
+  only:
+    - master
+

README-devel.md

+# TensorFlow image
+
+This is a preliminary image for TensorFlow. It is used by Zoe, the Container Analytics as a
+Service system to create a very simple TensorFlow batch service.
+
+Zoe can be found at: https://github.com/DistributedSystemsGroup/zoe
+
+## Setup
+
+The Dockerfile runs a start script that:
+
+* Launches your TensorFlow program until termination
+* Launches a TensorBoard instance to inspect the output of your program
+

build_all.sh

+#!/usr/bin/env bash
+
+set -e
+
+if [ ! -d docker ]; then
+  exit
+fi
+
+REPOSITORY=${REPOSITORY:-zapps}
+DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker-registry:5000}
+
+VERSION=${VERSION:-`date +%Y%m%d%H%M%S`}
+
+built_images=''
+for d in `find docker -mindepth 1 -maxdepth 1 -type d -printf '%f '`; do
+  pushd docker/${d}
+  docker build -t ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION} .
+  docker push ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}
+  popd
+  built_images+="${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}\n"
+done
+
+echo "-------------END SCRIPT-----------------"
+echo "Images built:"
+printf ${built_images}
+echo

docker/tensorflow/Dockerfile

+FROM ubuntu:14.04
+
+MAINTAINER Pietro Michiardi <Pietro.Michiardi@eurecom.fr>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+##################################################
+# BASELINE
+##################################################
+RUN apt-get update && apt-get install -y --force-yes --no-install-recommends \
+    python-pip \
+    python-dev \
+    software-properties-common \
+    curl \
+    pkg-config \
+    zip \
+    g++ \
+    zlib1g-dev \
+    unzip \
+    swig \
+    git \
+    swig \
+    && apt-get clean
+
+RUN python -m pip install --upgrade pip
+
+##################################################
+# JAVA
+##################################################
+RUN add-apt-repository ppa:webupd8team/java -y
+RUN /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
+RUN apt-get update && apt-get install -y oracle-java8-installer
+
+##################################################
+# TENSORFLOW
+##################################################
+# From sources
+
+# INSTALL BAZEL
+RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+
+RUN apt-get update && sudo apt-get install -y --force-yes --no-install-recommends bazel patch
+RUN apt-get upgrade -y --force-yes --no-install-recommends bazel
+
+# CLONE FROM GIT
+RUN git clone https://github.com/tensorflow/tensorflow
+WORKDIR /tensorflow
+
+##################################################
+# Make sure important packages are installed
+##################################################
+RUN pip install --upgrade numpy pandas matplotlib
+
+# COMPILE AND INSTALL PIP PACKAGE
+COPY files/conf.txt /tmp/conf.txt
+
+RUN ./configure < /tmp/conf.txt 
+RUN bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
+RUN pip install wheel
+RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
+RUN pip install /tmp/tensorflow_pkg/tensorflow-*.whl
+
+##################################################
+# GPy
+##################################################
+RUN pip install --upgrade GPy paramz
+
+##################################################
+# GPFlow
+##################################################
+WORKDIR /opt
+RUN git clone https://github.com/GPflow/GPflow.git
+WORKDIR /opt/GPflow
+RUN python setup.py install
+
+##################################################
+# SETUP ZOE APP
+##################################################
+# Configure environment
+ENV JAVA_HOME /usr/lib/jvm/java-8-oracle/
+ENV SHELL /bin/bash
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+# Configure container startup as root
+EXPOSE 6006
+WORKDIR /mnt/workspace
+ENV WS_DIR /mnt/workspace
+#COPY files/start-tf.sh /usr/local/bin/
+#CMD ["/usr/local/bin/start-tf.sh"]
+#RUN chmod 755 /usr/local/bin/start-tf.sh
+#ADD files/* /usr/local/bin/
+#RUN chmod +x /usr/local/bin/*
+#ENTRYPOINT ["/usr/local/bin/start-tf.sh"]
+
+
+

docker/tensorflow/files/conf.txt

+
+
+
+
+
+y
+
+
+
+

docker/tensorflow/files/start-tf.sh

+#!/usr/bin/env bash
+
+set -e
+
+cd $ZOE_WORKSPACE
+
+python "$@"

gen_json.py

+# Copyright (c) 2016, Daniele Venzano
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import sys
+import os
+
+APP_NAME = 'tensorflow'
+ZOE_APPLICATION_DESCRIPTION_VERSION = 3
+
+options = {
+    'core_limit': {
+        'value': 4,
+        'description': 'Core limit'
+    },
+    'memory_limit': {
+        'value': 4 * (1024**3),
+        'description': 'Memory limit (bytes)'
+    }
+}
+
+REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
+REPOSITORY = os.getenv("REPOSITORY", default="zapps")
+VERSION = os.getenv("VERSION", default="latest")
+
+GOOG_IMAGE = "gcr.io/tensorflow/tensorflow"
+CUSTOM_IMAGE = REGISTRY + '/' + REPOSITORY + '/tensorflow:' + VERSION
+
+def goog_tensorflow_service(memory_limit, core_limit):
+    """
+    :rtype: dict
+    """
+    service = {
+        'name': "jupyter",
+        'image': GOOG_IMAGE,
+        'monitor': True,
+        'resources': {
+            "memory": {
+                "min": memory_limit,
+                "max": memory_limit
+            },
+            "cores": {
+                "min": core_limit,
+                "max": core_limit
+            }
+        },
+        'ports': [
+            {
+                'name': "Tensorboard web interface",
+                'protocol': "tcp",
+                'port_number': 6006,
+                'url_template': "http://{ip_port}/"
+            },
+            {
+                'name': "Notebook web interface",
+                'protocol': "tcp",
+                'port_number': 8888,
+                'url_template': "http://{ip_port}/"
+            }
+        ],
+        'environment': [],
+        'volumes': [],
+        'command': None,
+        'total_count': 1,
+        'essential_count': 1,
+        'startup_order': 0
+    }
+
+    return service
+
+
+def custom_tensorflow_service(memory_limit, core_limit):
+    """
+    :rtype: dict
+    """
+    service = {
+        'name': "jupyter",
+        'image': CUSTOM_IMAGE,
+        'monitor': True,
+        'resources': {
+            "memory": {
+                "min": memory_limit,
+                "max": memory_limit
+            },
+            "cores": {
+                "min": core_limit,
+                "max": core_limit
+            }
+        },
+        'ports': [],
+        'environment': [],
+        'volumes': [],
+        'command': None,
+        'total_count': 1,
+        'essential_count': 1,
+        'startup_order': 0
+    }
+
+    return service
+
+
+
+if __name__ == '__main__':
+    app = {
+        'name': APP_NAME,
+        'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
+        'will_end': False,
+        'size': 512,
+        'services': [
+            goog_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
+        ]
+    }
+
+    json.dump(app, open("goog_tensorflow.json", "w"), sort_keys=True, indent=4)
+
+    app = {
+        'name': APP_NAME,
+        'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
+        'will_end': False,
+        'size': 512,
+        'services': [
+            custom_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
+        ]
+    }
+
+    json.dump(app, open("custom_tensorflow.json", "w"), sort_keys=True, indent=4)
+
+    print("ZApp written")
+