Commit 6f35dfce authored by Daniele Venzano's avatar Daniele Venzano

More scripts for testing HDFS

parent 87aa8b6a
#!/bin/bash
. utils/base.sh
docker -H $SWARM stats `cat state.zoe`
#!/bin/bash
set -e
. utils/base.sh
pushd ../utils
. base.sh
popd
for id in `cat state.zoe`; do
echo -n "Deleting container "
docker -H $SWARM rm -f $id
$DOCKER rm -f $id
done
rm state.zoe
......
#!/bin/bash
. utils/base.sh
pushd ../utils
. base.sh
popd
echo '<============================================================================>'
for id in `cat state.zoe`; do
docker -H $SWARM logs $id
$DOCKER logs $id
echo '<============================================================================>'
done
#!/bin/bash
set -e
pushd ../utils
. base.sh
popd
NAMENODE_IMAGE=$REGISTRY/zoerepo/hadoop-namenode
DATANODE_IMAGE=$REGISTRY/zoerepo/hadoop-datanode
DATANODE_COUNT=3
if [ -f state.zoe ]; then
echo "Error a cluster is already running"
exit 1
fi
PFX=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 3 | head -n 1`
echo -n "Starting HDFS NameNode container..."
NN_ID=`$DOCKER run --name namenode-$PFX -h namenode-$PFX -e NAMENODE_HOST=namenode-$PFX -d $NAMENODE_IMAGE`
echo "done"
echo $NN_ID > state.zoe
NN_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $NN_ID`
echo "NameNode is at $NN_IP:8020, web interface at http://$NN_IP:50070"
echo -n "Starting DataNode(s)... "
for w in `seq $DATANODE_COUNT`; do
DN_ID=`$DOCKER run --name datanode-$PFX-$w -h datanode-$PFX-$w -e NAMENODE_HOST=namenode-$PFX -d $DATANODE_IMAGE`
echo -n "$w "
echo $DN_ID >> state.zoe
done
echo done
echo "Creating nbuser home directory on HDFS"
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -mkdir /user
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -mkdir /user/nbuser
$DOCKER exec -it $NN_ID /opt/hadoop/bin/hdfs dfs -chown nbuser:users /user/nbuser
#!/bin/bash
pushd ../utils
. base.sh
popd
$DOCKER stats `cat state.zoe`
#!/bin/bash
pushd ../utils
. base.sh
popd
for id in `cat state.zoe`; do
echo -n "Deleting container "
$DOCKER rm -f $id
done
rm state.zoe
#!/bin/bash
pushd ../utils
. base.sh
popd
echo '<============================================================================>'
for id in `cat state.zoe`; do
$DOCKER logs $id
echo '<============================================================================>'
done
......@@ -2,13 +2,13 @@
set -e
. utils/base.sh
pushd ../utils
. base.sh
popd
MASTER_IMAGE=$REGISTRY/zoerepo/spark-master
WORKER_IMAGE=$REGISTRY/zoerepo/spark-worker
NOTEBOOK_IMAGE=$REGISTRY/zoerepo/spark-ipython-notebook
echo $MASTER_IMAGE
NOTEBOOK_IMAGE=$REGISTRY/zoerepo/spark-jupyter-notebook
WORKER_COUNT=3
WORKER_RAM=7g
......@@ -17,34 +17,32 @@ WORKER_CORES=4
EXECUTOR_RAM=6g
SPARK_SHELL_OPTIONS=""
if [ -f state.zoe ]; then
echo "Error a cluster is already running"
exit 1
fi
echo -n "Starting Spark master container..."
MASTER_ID=`docker -H $SWARM run -d $MASTER_IMAGE`
MASTER_ID=`$DOCKER run -d -p 8080:8080 $MASTER_IMAGE`
echo "done"
echo $MASTER_ID > state.zoe
MASTER_IP=`docker -H $SWARM inspect --format '{{ .NetworkSettings.IPAddress }}' $MASTER_ID`
MASTER_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $MASTER_ID`
echo "Spark master is at http://$MASTER_IP:8080"
echo -n "Starting workers... "
for w in `seq $WORKER_COUNT`; do
WORKER_ID=`docker -H $SWARM run -e SPARK_MASTER_IP=$MASTER_IP -e SPARK_WORKER_RAM=$WORKER_RAM -e SPARK_WORKER_CORES=$WORKER_CORES -m $CONTAINER_RAM -d $WORKER_IMAGE`
WORKER_ID=`$DOCKER run -e SPARK_MASTER_IP=$MASTER_IP -e SPARK_WORKER_RAM=$WORKER_RAM -e SPARK_WORKER_CORES=$WORKER_CORES -m $CONTAINER_RAM -d $WORKER_IMAGE`
echo -n "$w "
echo $WORKER_ID >> state.zoe
done
echo done
echo -n "Starting notebook..."
NB_ID=`docker -H $SWARM run -e SPARK_MASTER_IP=$MASTER_IP -e SPARK_EXECUTOR_RAM=$EXECUTOR_RAM -e SPARK_OPTIONS=$SPARK_SHELL_OPTIONS -d $NOTEBOOK_IMAGE`
NB_ID=`$DOCKER run -e SPARK_MASTER=spark://$MASTER_IP:7077 -e SPARK_EXECUTOR_RAM=$EXECUTOR_RAM -p 8888:8888 -p 4040:4040 -d $NOTEBOOK_IMAGE`
echo "done"
echo $NB_ID >> state.zoe
NB_IP=`docker -H $SWARM inspect --format '{{ .NetworkSettings.IPAddress }}' $NB_ID`
NB_IP=`$DOCKER inspect --format '{{ .NetworkSettings.IPAddress }}' $NB_ID`
echo "iPython notebook is available at http://$NB_IP:8888"
#!/bin/bash
pushd ../utils
. base.sh
popd
$DOCKER stats `cat state.zoe`
#!/bin/bash
source utils/base.sh
pushd utils
source base.sh
popd
docker -H $SWARM info
$DOCKER info
#!/bin/bash
source utils/base.sh
pushd utils
source base.sh
popd
docker -H $SWARM ps -a
$DOCKER ps -a
ZK_HOSTS="bf1.bigfoot.eurecom.fr:2181,bf5.bigfoot.eurecom.fr:2181,bf11.bigfoot.eurecom.fr"
SWARM=`utils/find_swarm_master.py $ZK_HOSTS`
SWARM=`./find_swarm_master.py $ZK_HOSTS`
REGISTRY=10.1.0.1:5000
if [ z"$1" = z"--local" ]; then
DOCKER="sudo docker"
else
DOCKER="docker -H $SWARM"
fi
#!/usr/bin/python
#!/usr/bin/python3
import sys
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment