From eeffdfa200fe834e53f13b8692632baac6ee2dfb Mon Sep 17 00:00:00 2001
From: Raphael Defosseux <raphael.defosseux@eurecom.fr>
Date: Wed, 20 Feb 2019 18:13:43 +0100
Subject: [PATCH] CI: adding much needed lock mechanism when creating VM.   --
 should prevent VM-creation crashes during test stages

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
---
 ci-scripts/buildOnVM.sh   | 17 ++++++++++++-----
 ci-scripts/createVM.sh    | 31 +++++++++++++++++++++++++++++++
 ci-scripts/runTestOnVM.sh |  9 ++++++++-
 3 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh
index f9fb3b776f2..ad77d39eb78 100755
--- a/ci-scripts/buildOnVM.sh
+++ b/ci-scripts/buildOnVM.sh
@@ -94,14 +94,21 @@ function build_on_vm {
         echo "############################################################"
         echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base"
         echo "############################################################"
+        acquire_vm_create_lock
         uvt-kvm create $VM_NAME release=xenial --memory $VM_MEMORY --cpu $VM_CPU --unsafe-caching --template ci-scripts/template-host.xml
-    fi
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $VM_NAME --insecure
 
-    echo "Waiting for VM to be started"
-    uvt-kvm wait $VM_NAME --insecure
+        VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
+        echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+        release_vm_create_lock
+    else
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $VM_NAME --insecure
 
-    VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
-    echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+        VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
+        echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+    fi
 
     echo "############################################################"
     echo "Copying GIT repo into VM ($VM_NAME)" 
diff --git a/ci-scripts/createVM.sh b/ci-scripts/createVM.sh
index 192008d8672..e693b07b913 100755
--- a/ci-scripts/createVM.sh
+++ b/ci-scripts/createVM.sh
@@ -49,6 +49,35 @@ function create_usage {
     echo ""
 }
 
+function acquire_vm_create_lock {
+    local FlockFile="/tmp/vmclone.lck"
+    local unlocked="0"
+    touch ${FlockFile} 2>/dev/null
+    if [[ $? -ne 0 ]]
+    then
+        echo "Cannot access lock file ${FlockFile}"
+        exit 2
+    fi
+    while [ $unlocked -eq 0 ]
+    do
+        exec 5>${FlockFile}
+        flock -nx 5
+        if [[ $? -ne 0 ]]
+        then
+            echo "Another instance of VM creation is running"
+            sleep 10
+        else
+            unlocked="1"
+        fi
+    done
+    chmod 666 ${FlockFile} 2>/dev/null
+}
+
+function release_vm_create_lock {
+    local FlockFile="/tmp/vmclone.lck"
+    rm -Rf ${FlockFile}
+}
+
 function create_vm {
     echo "############################################################"
     echo "OAI CI VM script"
@@ -60,10 +89,12 @@ function create_vm {
     echo "############################################################"
     echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base"
     echo "############################################################"
+    acquire_vm_create_lock
     uvt-kvm create $VM_NAME release=xenial --memory $VM_MEMORY --cpu $VM_CPU --unsafe-caching --template ci-scripts/template-host.xml
     echo "Waiting for VM to be started"
     uvt-kvm wait $VM_NAME --insecure
 
     VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
     echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+    release_vm_create_lock
 }
diff --git a/ci-scripts/runTestOnVM.sh b/ci-scripts/runTestOnVM.sh
index 9aa6606eea8..b3eac341a91 100755
--- a/ci-scripts/runTestOnVM.sh
+++ b/ci-scripts/runTestOnVM.sh
@@ -317,11 +317,18 @@ function install_epc_on_vm {
         echo "############################################################"
         echo "Creating test EPC VM ($LOC_EPC_VM_NAME) on Ubuntu Cloud Image base"
         echo "############################################################"
+        acquire_vm_create_lock
         uvt-kvm create $LOC_EPC_VM_NAME release=xenial --unsafe-caching
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $LOC_EPC_VM_NAME --insecure
+        release_vm_create_lock
+    else
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $LOC_EPC_VM_NAME --insecure
     fi
 
-    uvt-kvm wait $LOC_EPC_VM_NAME --insecure
     local LOC_EPC_VM_IP_ADDR=`uvt-kvm ip $LOC_EPC_VM_NAME`
+
     echo "$LOC_EPC_VM_NAME has for IP addr = $LOC_EPC_VM_IP_ADDR"
     scp -o StrictHostKeyChecking=no /etc/apt/apt.conf.d/01proxy ubuntu@$LOC_EPC_VM_IP_ADDR:/home/ubuntu
 
-- 
GitLab