Update image and gitlab CI

.gitlab-ci.yml

@@ -19,9 +19,6 @@ build:json:
 build:hub:
   stage: build
   script:
-    - docker pull tensorflow/magenta
-    - docker pull tensorflow/syntaxnet
-    - docker pull gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3
     - docker pull gcr.io/tensorflow/tensorflow:1.3.0-py3
 
 test:json:
@@ -30,14 +27,11 @@ test:json:
   before_script:
     - pip install requests
   script:
-    - python /scripts/validate.py ${ZOE_VALIDATION_URL} *google.json
+    - python /scripts/validate.py ${ZOE_VALIDATION_URL} tf-google.json
 
 pull:all:
   stage: pull
   script:
-    - /scripts/distribute_docker_image.sh tensorflow/magenta
-    - /scripts/distribute_docker_image.sh tensorflow/syntaxnet
-    - /scripts/distribute_docker_image.sh gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3
     - /scripts/distribute_docker_image.sh gcr.io/tensorflow/tensorflow:1.3.0-py3
 
 deploy:json:

README-custom.md

-# Batch TensorFlow
-
-Stand-alone batch TensorFlow version 1.3 for Python 3, recompiled with instructions for modern x86_64 Intel CPUs.
-
-This ZApp expects as parameter the command to run, relative to the workspace. For example, if you want to run
-the script `~/algo/run.py`, you can write `python3 ./algo/run.py`.
-

README-devel.md

-# TensorFlow image
-
-This is a preliminary image for TensorFlow. It is used by Zoe, the Container Analytics as a
-Service system to create a very simple TensorFlow batch service.
-
-Zoe can be found at: https://github.com/DistributedSystemsGroup/zoe
-
-## Setup
-
-The Dockerfile runs a start script that:
-
-* Launches your TensorFlow program until termination
-* Launches a TensorBoard instance to inspect the output of your program
-

README-magenta.md

-# Tensorflow Magenta
-
-Unmodified [TensorFlow Magenta](https://magenta.tensorflow.org) as generated by Google.
-
-Please note that you need to get the Notebook key from the service logs to be able to access the Notebook.
-

README-syntaxnet.md

-# SyntaxNet TensorFlow
-
-URL: [https://github.com/tensorflow/models/tree/master/syntaxnet](https://github.com/tensorflow/models/tree/master/syntaxnet)
-
-DRAGNN (Dynamic Recurrent Acyclic Graphical Neural Networks) is a framework for building multi-task, dynamically connected computation graphs.
-In practice, it uses SyntaxNet as its backend to perform several NLP tasks in a joint model (publicly released models include segmentation + POS tagging + syntactic parsing).
-
-Please note that you need to get the Notebook key from the service logs to be able to access the Notebook.
-

build_all.sh

-#!/usr/bin/env bash
-
-set -e
-
-if [ ! -d docker ]; then
-  exit
-fi
-
-REPOSITORY=${REPOSITORY:-zapps}
-DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker-registry:5000}
-
-VERSION=${VERSION:-`date +%Y%m%d%H%M%S`}
-
-built_images=''
-for d in `find docker -mindepth 1 -maxdepth 1 -type d -printf '%f '`; do
-  pushd docker/${d}
-  docker build -t ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION} .
-  docker push ${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}
-  popd
-  built_images+="${DOCKER_REGISTRY}/${REPOSITORY}/${d}:${VERSION}\n"
-done
-
-echo "-------------END SCRIPT-----------------"
-echo "Images built:"
-printf ${built_images}
-echo

docker/tensorflow/Dockerfile

-FROM ubuntu:14.04
-
-MAINTAINER Pietro Michiardi <Pietro.Michiardi@eurecom.fr>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-##################################################
-# BASELINE
-##################################################
-RUN apt-get update && apt-get install -y --force-yes --no-install-recommends \
-    python3-pip \
-    python3-dev \
-    software-properties-common \
-    curl \
-    pkg-config \
-    zip \
-    g++ \
-    zlib1g-dev \
-    unzip \
-    swig \
-    git \
-    swig \
-    && apt-get clean
-
-RUN python3 -m pip install --upgrade pip
-
-##################################################
-# JAVA
-##################################################
-RUN add-apt-repository ppa:webupd8team/java -y
-RUN /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-RUN apt-get update && apt-get install -y oracle-java8-installer
-
-##################################################
-# TENSORFLOW
-##################################################
-# From sources
-
-# INSTALL BAZEL
-RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
-RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
-
-RUN apt-get update && sudo apt-get install -y --force-yes --no-install-recommends bazel patch
-RUN apt-get upgrade -y --force-yes --no-install-recommends bazel
-
-##################################################
-# Make sure important packages are installed
-##################################################
-RUN pip3 install --upgrade numpy pandas matplotlib
-
-# Running bazel inside a `docker build` command causes trouble, cf:
-#   https://github.com/bazelbuild/bazel/issues/134
-# The easiest solution is to set up a bazelrc file forcing --batch.
-RUN echo "startup --batch" >>/etc/bazel.bazelrc
-# Similarly, we need to workaround sandboxing issues:
-#   https://github.com/bazelbuild/bazel/issues/418
-RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
-    >>/etc/bazel.bazelrc
-# Install the most recent bazel release.
-ENV BAZEL_VERSION 0.5.0
-WORKDIR /
-RUN mkdir /bazel && \
-    cd /bazel && \
-    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
-    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
-    chmod +x bazel-*.sh && \
-    ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
-    cd / && \
-    rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
-
-# Download and build TensorFlow.
-
-RUN git clone https://github.com/tensorflow/tensorflow.git && \
-    cd tensorflow && \
-    git checkout r1.3
-WORKDIR /tensorflow
-
-# TODO(craigcitro): Don't install the pip package, since it makes it
-# more difficult to experiment with local changes. Instead, just add
-# the built directory to the path.
-
-ENV CI_BUILD_PYTHON python3
-
-RUN pip3 --no-cache-dir install wheel
-
-RUN sed -i '\@https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz@d' tensorflow/workspace.bzl
-RUN tensorflow/tools/ci_build/builds/configured CPU \
-    bazel build -c opt --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
-        tensorflow/tools/pip_package:build_pip_package && \
-    bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
-    pip3 --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
-    rm -rf /tmp/pip && \
-    rm -rf /root/.cache
-# Clean up pip wheel and Bazel cache when done.
-
-# COMPILE AND INSTALL PIP PACKAGE
-#COPY files/conf.txt /tmp/conf.txt
-
-#RUN ./configure < /tmp/conf.txt 
-#RUN bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
-#RUN pip install wheel
-#RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
-#RUN pip install /tmp/tensorflow_pkg/tensorflow-*.whl
-
-##################################################
-# GPy
-##################################################
-RUN pip3 install --upgrade GPy paramz
-
-##################################################
-# GPFlow
-##################################################
-WORKDIR /opt
-RUN git clone https://github.com/GPflow/GPflow.git
-WORKDIR /opt/GPflow
-RUN python3 setup.py install
-
-##################################################
-# SETUP ZOE APP
-##################################################
-# Configure environment
-ENV JAVA_HOME /usr/lib/jvm/java-8-oracle/
-ENV SHELL /bin/bash
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-# Configure container startup as root
-EXPOSE 6006
-WORKDIR /mnt/workspace
-ENV WS_DIR /mnt/workspace
-#COPY files/start-tf.sh /usr/local/bin/
-#CMD ["/usr/local/bin/start-tf.sh"]
-#RUN chmod 755 /usr/local/bin/start-tf.sh
-#ADD files/* /usr/local/bin/
-#RUN chmod +x /usr/local/bin/*
-#ENTRYPOINT ["/usr/local/bin/start-tf.sh"]
-
-
-

docker/tensorflow/files/conf.txt

-
-
-
-
-
-y
-
-
-
-

docker/tensorflow/files/start-tf.sh

-#!/usr/bin/env bash
-
-set -e
-
-cd $ZOE_WORKSPACE
-
-python "$@"

gen_json_google.py

-# Copyright (c) 2016, Daniele Venzano
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import sys
-import os
-
-APPS = [
-    ('tf-google', "gcr.io/tensorflow/tensorflow:1.3.0-py3", False),
-    ('stnet-google', "tensorflow/syntaxnet", False),
-    ('mag-google', "tensorflow/magenta", False),
-    ('tf-google-gpu', "gcr.io/tensorflow/tensorflow:1.3.0-gpu-py3", True)
-]
-
-ZOE_APPLICATION_DESCRIPTION_VERSION = 3
-
-options = {
-    'core_limit': {
-        'value': 4,
-        'description': 'Core limit'
-    },
-    'memory_limit': {
-        'value': 32 * (1024**3),
-        'description': 'Memory limit (bytes)'
-    }
-}
-
-REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
-REPOSITORY = os.getenv("REPOSITORY", default="zapps")
-VERSION = os.getenv("VERSION", default="latest")
-
-def goog_tensorflow_service(memory_limit, core_limit, image, gpu=False):
-    """
-    :rtype: dict
-    """
-    service = {
-        'name': "tf-jupyter",
-        'image': image,
-        'monitor': True,
-        'resources': {
-            "memory": {
-                "min": memory_limit,
-                "max": memory_limit
-            },
-            "cores": {
-                "min": core_limit,
-                "max": core_limit
-            }
-        },
-        'ports': [
-            {
-                'name': "Tensorboard web interface",
-                'protocol': "tcp",
-                'port_number': 6006,
-                'url_template': "http://{ip_port}/"
-            },
-            {
-                'name': "Notebook web interface",
-                'protocol': "tcp",
-                'port_number': 8888,
-                'url_template': "http://{ip_port}/"
-            }
-        ],
-        'environment': [],
-        'volumes': [],
-        'command': None,
-        'total_count': 1,
-        'essential_count': 1,
-        'startup_order': 0,
-        'replicas': 1
-    }
-    if gpu:
-        service['labels'] = ['gpu']
-        service['environment'] = [['NVIDIA_VISIBLE_DEVICES', 'all']]
-
-    return service
-
-
-if __name__ == '__main__':
-    for app_name, image, gpu in APPS:
-        app = {
-            'name': app_name,
-            'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
-            'will_end': False,
-            'size': 512,
-            'services': [
-                goog_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"], image, gpu)
-            ]
-        }
-
-        json.dump(app, open(app_name + ".json", "w"), sort_keys=True, indent=4)
-
-    print("ZApps written")
-

gen_json_standalone.py

-# Copyright (c) 2016, Daniele Venzano
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import sys
-import os
-
-APP_NAME = 'tf-standalone'
-ZOE_APPLICATION_DESCRIPTION_VERSION = 3
-
-options = {
-    'core_limit': {
-        'value': 4,
-        'description': 'Core limit'
-    },
-    'memory_limit': {
-        'value': 4 * (1024**3),
-        'description': 'Memory limit (bytes)'
-    }
-}
-
-REGISTRY = os.getenv("DOCKER_REGISTRY", default="docker-engine:5000")
-REPOSITORY = os.getenv("REPOSITORY", default="zapps")
-VERSION = os.getenv("VERSION", default="latest")
-
-CUSTOM_IMAGE = REGISTRY + '/' + REPOSITORY + '/tensorflow:' + VERSION
-
-
-def custom_tensorflow_service(memory_limit, core_limit):
-    """
-    :rtype: dict
-    """
-    service = {
-        'name': "tf",
-        'image': CUSTOM_IMAGE,
-        'monitor': True,
-        'resources': {
-            "memory": {
-                "min": memory_limit,
-                "max": memory_limit
-            },
-            "cores": {
-                "min": core_limit,
-                "max": core_limit
-            }
-        },
-        'ports': [],
-        'environment': [],
-        'volumes': [],
-        'command': None,
-        'total_count': 1,
-        'essential_count': 1,
-        'startup_order': 0,
-        'replicas': 1
-    }
-
-    return service
-
-
-
-if __name__ == '__main__':
-    app = {
-        'name': APP_NAME,
-        'version': ZOE_APPLICATION_DESCRIPTION_VERSION,
-        'will_end': False,
-        'size': 512,
-        'services': [
-            custom_tensorflow_service(options["memory_limit"]["value"], options["core_limit"]["value"])
-        ]
-    }
-
-    json.dump(app, open("custom_tensorflow.json", "w"), sort_keys=True, indent=4)
-
-    print("ZApp written")
-

manifest.json

@@ -8,13 +8,6 @@
             "readable_descr": "README-goog.md",
             "parameters": []
         },
-        {
-            "category": "Machine learning",
-            "name": "Google TensorFlow notebook with GPU",
-            "description": "tf-google-gpu.json",
-            "readable_descr": "README-goog.md",
-            "parameters": []
-        },
         {
             "category": "Non-interactive",
             "name": "Google TensorFlow batch",
@@ -30,20 +23,6 @@
                     "default": "./my-tf-app/main.py"
                 }
             ]
-        },
-        {
-            "category": "Tutorials and examples",
-            "name": "DRAGNN SyntaxNet model",
-            "description": "stnet-google.json",
-            "readable_descr": "README-syntaxnet.md",
-            "parameters": []
-        },
-        {
-            "category": "Tutorials and examples",
-            "name": "Magenta model",
-            "description": "mag-google.json",
-            "readable_descr": "README-magenta.md",
-            "parameters": []
         }
     ]
 }