Commit 8d429a4c authored by Daniele Venzano's avatar Daniele Venzano

Remove outdated scripts

parent c1b171ee
#!/bin/bash
DOCKER=`which docker`
if [ -z "$DOCKER" ]; then
echo "Docker is missing"
fi
DOCKER_VERSION=`$DOCKER --version`
if [ z"$DOCKER_VERSION" != z"Docker version 1.8.3, build f4bf5c7" ]; then
echo "Wrong Docker version, please use 1.8.3"
fi
if ! which pip > /dev/null; then
if ! dpkg -l | grep -q kazoo; then
echo "Please install python-kazoo"
fi
fi
if ! pip freeze | grep -q kazoo; then
echo "Please install python-kazoo"
fi
if ! route -n | grep -q 192.168.46; then
echo "Please add routing: sudo route add -net 192.168.46.0/24 dev eth0"
fi
if ! grep -q -e '\<bigfoot.eurecom.fr\>' /etc/resolv.conf; then
echo "Please add bigfoot.eurecom.fr to the search list in /etc/resolv.conf"
fi
echo "If nothing is printed above this line, all dependencies are ok"
#!/bin/bash
NETWORK_NAME=hdfs
NETWORK_ID=eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792
VOLUME=/mnt/hdfs/datanode
SWARM=192.168.45.252:2380
IMAGE=192.168.45.252:5000/zoerepo/hadoop-datanode
#NODES="bf12 bf13 bf14 bf15"
NODES="bf15"
for node in $NODES; do
docker -H $SWARM run -i -t -d --name hdfs-datanode-$node \
-h hdfs-datanode-$node \
-e NAMENODE_HOST=hdfs-namenode \
-e constraint:node==$node \
-m 1g \
--net $NETWORK_NAME \
-v $VOLUME:/mnt/datanode \
$IMAGE
done
#!/bin/bash
NETWORK_NAME=hdfs
NETWORK_ID=eeef9754c16790a29d5210c5d9ad8e66614ee8a6229b6dc6f779019d46cec792
VOLUME_PATH=/mnt/hdfs/namenode
SWARM=192.168.45.252:2380
IMAGE=192.168.45.252:5000/zoerepo/hadoop-namenode
docker -H $SWARM run -i -t -d --name hdfs-namenode \
-h hdfs-namenode \
-e NAMENODE_HOST=hdfs-namenode \
-e constraint:node==bf12 \
-m 2g \
-v $VOLUME_PATH:/mnt/namenode \
--net $NETWORK_NAME \
$IMAGE
#!/bin/bash
pushd ../utils
. base.sh
popd
for id in `cat state.zoe`; do
echo -n "Deleting container "
$DOCKER rm -f $id
done
rm state.zoe
#!/bin/bash
pushd ../utils
. base.sh
popd
echo '<============================================================================>'
for id in `cat state.zoe`; do
$DOCKER logs $id
echo '<============================================================================>'
done
#!/bin/bash
set -e
pushd ../utils
. base.sh
popd
NAMENODE_IMAGE=$REGISTRY/zoerepo/hadoop-namenode
DATANODE_IMAGE=$REGISTRY/zoerepo/hadoop-datanode
DATANODE_COUNT=3
if [ -f state.zoe ]; then
echo "Error a cluster is already running"
exit 1
fi
PFX=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 3 | head -n 1`
echo -n "Starting HDFS NameNode container..."
NN_ID=`$DOCKER run --name namenode-$PFX -h namenode-$PFX -e NAMENODE_HOST=namenode-$PFX -d $NAMENODE_IMAGE`
echo "done"
echo $NN_ID > state.zoe
NN_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $NN_ID`
echo "NameNode is at $NN_IP:8020, web interface at http://$NN_IP:50070"
echo -n "Starting DataNode(s)... "
for w in `seq $DATANODE_COUNT`; do
DN_ID=`$DOCKER run --name datanode-$PFX-$w -h datanode-$PFX-$w -e NAMENODE_HOST=namenode-$PFX -d $DATANODE_IMAGE`
echo -n "$w "
echo $DN_ID >> state.zoe
done
echo done
echo "Creating nbuser home directory on HDFS"
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -mkdir /user
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -mkdir /user/nbuser
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -chown nbuser:users /user/nbuser
#!/bin/bash
pushd ../utils
. base.sh
popd
$DOCKER stats `cat state.zoe`
#!/usr/bin/env bash
for i in `seq -w 1 50`; do ./zoe.py user-new --name group$i --pass group$i --role guest; done
./zoe.py pre-app-export spark_jupyter_notebook_lab > /tmp/zoeapp.json
for i in `seq -w 1 50`; do
export ZOE_USER=group$i
export ZOE_PASS=group$i
./zoe.py app-new /tmp/zoeapp.json
done
for i in `seq -w 1 50`; do
export ZOE_USER=group$i
export ZOE_PASS=group$i
./zoe.py start spark-jupyter-lab lab-session
done
#!/bin/bash
pushd ../utils
. base.sh
popd
for id in `cat state.zoe`; do
echo -n "Deleting container "
$DOCKER rm -f $id
done
rm state.zoe
#!/bin/bash
pushd ../utils
. base.sh
popd
echo '<============================================================================>'
for id in `cat state.zoe`; do
$DOCKER logs $id
echo '<============================================================================>'
done
#!/bin/bash
set -e
pushd ../utils
. base.sh
popd
MASTER_IMAGE=$REGISTRY/zoerepo/spark-master
WORKER_IMAGE=$REGISTRY/zoerepo/spark-worker
NOTEBOOK_IMAGE=$REGISTRY/zoerepo/spark-jupyter-notebook
WORKER_COUNT=3
WORKER_RAM=7g
CONTAINER_RAM=8g
WORKER_CORES=4
EXECUTOR_RAM=6g
if [ -f state.zoe ]; then
echo "Error a cluster is already running"
exit 1
fi
echo -n "Starting Spark master container..."
MASTER_ID=`$DOCKER run -d -p 8080:8080 $MASTER_IMAGE`
echo "done"
echo $MASTER_ID > state.zoe
MASTER_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $MASTER_ID`
echo "Spark master is at http://$MASTER_IP:8080"
echo -n "Starting workers... "
for w in `seq $WORKER_COUNT`; do
WORKER_ID=`$DOCKER run -e SPARK_MASTER_IP=$MASTER_IP -e SPARK_WORKER_RAM=$WORKER_RAM -e SPARK_WORKER_CORES=$WORKER_CORES -m $CONTAINER_RAM -d $WORKER_IMAGE`
echo -n "$w "
echo $WORKER_ID >> state.zoe
done
echo done
echo -n "Starting notebook..."
NB_ID=`$DOCKER run -e SPARK_MASTER=spark://$MASTER_IP:7077 -e SPARK_EXECUTOR_RAM=$EXECUTOR_RAM -p 8888:8888 -p 4040:4040 -d $NOTEBOOK_IMAGE`
echo "done"
echo $NB_ID >> state.zoe
NB_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $NB_ID`
echo "iPython notebook is available at http://$NB_IP:8888"
#!/bin/bash
pushd ../utils
. base.sh
popd
$DOCKER stats `cat state.zoe`
#!/bin/bash
pushd utils
source base.sh
popd
$DOCKER info
#!/bin/bash
pushd utils
source base.sh
popd
$DOCKER ps -a
ZK_HOSTS="bf1.bigfoot.eurecom.fr:2181,bf5.bigfoot.eurecom.fr:2181,bf11.bigfoot.eurecom.fr"
SWARM=`./find_swarm_master.py $ZK_HOSTS`
REGISTRY=10.1.0.1:5000
if [ z"$1" = z"--local" ]; then
DOCKER="sudo docker"
else
DOCKER="docker -H $SWARM"
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment