Initial draft on new pipeline strategy

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci fix using docker-compose

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci fix

Signed-off-by: Mohammed Ismail <mohammed.ismail@openairinterface.org>

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -38,6 +38,9 @@ def cn_ci_host = params.Host_CN_CI_Server
 def cn_ci_resource = params.DockerContainers
 def ds_tester_ci_resource = params.DsTester
 
+// Location of the DsTester workspace
+def dsTestFrameworkLocation = params.dsTestFrameworkLocation
+
 // When triggered by upstream, specify which tag to use
 def upstreamTagToUse = params.upstreamTagToUse
 
@@ -58,7 +61,7 @@ def upstreamEvent = false
 
 // Default tags / branches  --> could be passed on by upstream job or by PR content
 def nrfTag = 'develop'
-def nrfBranch = 'develop'
+def nrfBranch = 'dockerfile'
 def amfTag = 'develop'
 def amfBranch = 'develop'
 def smfTag = 'develop'
@@ -200,40 +203,111 @@ pipeline {
             sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
             sh "mkdir -p archives DS-TEST-RESULTS"
           }
+          // Verify that the images are available
+          try {
+            sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+          } catch (Exception e) {
+            error "OAI NRF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_AMF_TAG: oai-amf:' + amfTag +'" > archives/oai_amf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
+          } catch (Exception e) {
+            error "OAI AMF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_SMF_TAG: oai-smf:' + smfTag +'" > archives/oai_smf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
+          } catch (Exception e) {
+            error "OAI SMF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_SPGWU_TAG: oai-spgwu-tiny:' + spgwuTag +'" > archives/oai_spgwu_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
+          } catch (Exception e) {
+            error "OAI SPGW-U-Tiny Image tag to test does not exist!"
+          }
         }
       }
     }
-    stage ('Deploy Whole EPC') {
-      stages {
-        stage ('Create Public Network') {
-          steps {
-            script {
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=CreateNetworks', new_host_flag, new_host_user, new_host)
-            }
+    stage ('Deploy Whole 5G Core Network') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mDeploy CN5G in idle mode\u001B[0m'
+          // Prepare all needed files for docker-compose
+          // First put all correct tags to test
+          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" -e "s#AMF_IMAGE_TAG#' + amfTag + '#" -e "s#SMF_IMAGE_TAG#' + smfTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
+          // Entrypoints are modified to be inactive for dsTester framework
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-nrf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/nrf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-amf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/amf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-smf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/smf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-upf-equivalent/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/upf-entrypoint-sleep.sh'
+          sh 'chmod 775 ci-scripts/dsTesterDockerCompose/*entrypoint-sleep.sh'
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose up -d  > ../../archives/compose_5gcn_up.log 2>&1'
+            sh 'sleep 100'
+            // Do a check on number of healthy containers
+            // 5 == mysql + nrf + amf + smf + upf(spgwu-tiny)
+            ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
+            ret = ret.trim()
+            if (ret != '5') {
+              error "Deployment went wrong!"
+            }          
           }
         }
-        stage ('Deploy Containers') {
-          steps {
-            script {
-              // Deploy and configure MySQL Server
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployMySqlServer', new_host_flag, new_host_user, new_host)
-              // Deploy and configure AMF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployAMF --tag=' + amfTag, new_host_flag, new_host_user, new_host)
-              // Deploy and configure SMF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeploySMF --tag=' + smfTag, new_host_flag, new_host_user, new_host)
-              // Deploy and configure UPF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployUPF --tag=' + spgwuTag, new_host_flag, new_host_user, new_host)
+      }
+      post {
+        always {
+          script {
+            // Check status on mysql.
+            try {
+              sh 'docker exec -it cicd-mysql-svr /bin/bash -c "mysqladmin -u root --password=linux ping" > archives/mysql_status.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-mysql-svr >> archives/mysql_status.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/mysql_status.log'
             }
-          }
-          post {
-            always {
-              script {
-                copyFrom2ndServer('archives/*_config.log', 'archives', new_host_flag, new_host_user, new_host)
-                sh 'python3 ./ci-scripts/generateHtmlReportDeployment.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
-              }
+            // Do docker logs to recover the configuration results
+            try {
+              sh 'docker logs cicd-oai-nrf > archives/nrf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf >> archives/nrf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/nrf_config.log'
+            }
+            try {
+              sh 'docker logs cicd-oai-amf > archives/amf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-amf >> archives/amf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/amf_config.log'
+            }
+            try {
+              sh 'docker logs cicd-oai-smf > archives/smf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-smf >> archives/smf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: OK" >> archives/smf_config.log'
+            }
+            try {
+              sh 'docker logs cicd-oai-upf > archives/spgwu_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-upf >> archives/spgwu_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/spgwu_config.log'
             }
           }
         }
+        success {
+          script {
+            sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log'
+          }
+        }
+        unsuccessful {
+          script {
+            sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log'
+          }
+        }
       }
     }
     stage ('Check with DS Tester') {
@@ -241,32 +315,20 @@ pipeline {
       steps {
         lock (ds_tester_ci_resource) {
           script {
-//            myShCmd('cd /home/oaici/CI-dev-ops-cn5g && git clean -x -d -f > /dev/null', new_host_flag, new_host_user, new_host)
-//            myShCmdWithLog('cd /home/oaici/CI-dev-ops-cn5g/scripts && CI_ENV=True SRC_BASE_DIR=/tmp/CI-CN5G-FED ./run-5gc.bash -pt --shark --detach --get-results --5g', 'archives/run-5g-dstester.log', new_host_flag, new_host_user, new_host)
-//            copyFrom2ndServer('DS-TEST-RESULTS/status.txt', 'DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-//            copyFrom2ndServer('DS-TEST-RESULTS/*.tar', 'DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-            sh 'cd /home/oaici/CI-dev-ops-cn5g && git clean -x -d -f > /dev/null'
-            sh 'cd /home/oaici/CI-dev-ops-cn5g/scripts && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && CI_ENV=True SRC_BASE_DIR=' + WORKSPACE + ' ./run-5gc.bash -pt --shark --detach --get-results --5g > ' + WORKSPACE + '/archives/run-5g-dstester.log 2>&1'
-            sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
+            sh 'cd ' + dsTestFrameworkLocation + ' && git clean -x -d -f > /dev/null'
+            sh 'cd ' + dsTestFrameworkLocation + '/scripts && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && CI_ENV=True SRC_BASE_DIR=' + WORKSPACE + ' ./run-5gc.bash -pt --shark --detach --get-results --5g > ' + WORKSPACE + '/archives/run-5g-dstester.log 2>&1'
+            sh 'cd ' + dsTestFrameworkLocation + ' && git stash > /dev/null'
+            sh 'cd ' + dsTestFrameworkLocation + ' && git stash clear > /dev/null'
           }
         }
       }
     }
-    stage ('Undeploy EPC') {
-      stages {
-        stage ('Undeploy Containers') {
-          steps {
-            script {
-              // Remove the containers
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveAllContainers', new_host_flag, new_host_user, new_host)
-            }
-          }
-        }
-        stage ('Delete Public Network') {
-          steps {
-            script {
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveNetworks', new_host_flag, new_host_user, new_host)
-            }
+    stage ('Undeploy 5G-CN') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mUn-Deploy CN5G\u001B[0m'
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1'
           }
         }
       }
@@ -275,20 +337,20 @@ pipeline {
   post {
     always {
       script { 
-        // Removing all containers
-        myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveAllContainers', new_host_flag, new_host_user, new_host)
-        // Removing the network
-        myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveNetworks', new_host_flag, new_host_user, new_host)
+        // Remove any leftover containers/networks
+        dir('ci-scripts/dsTesterDockerCompose') {
+          sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1'
+        }
+        // Generating the HTML report
+        sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
+
         // Zipping all archived log files
         sh "zip -r -qq cn5g_fed_docker_logs.zip archives DS-TEST-RESULTS/*.tar DS-TEST-RESULTS/status.txt"
         if (fileExists('cn5g_fed_docker_logs.zip')) {
           archiveArtifacts artifacts: 'cn5g_fed_docker_logs.zip'
         }
-        if (fileExists('deploy_results_oai_cn5g.html')) {
-          archiveArtifacts artifacts: 'deploy_results_oai_cn5g.html'
-        }
-        if (fileExists('ds_tester_results_oai_cn5g.html')) {
-          archiveArtifacts artifacts: 'ds_tester_results_oai_cn5g.html'
+        if (fileExists('test_results_oai_cn5g.html')) {
+          archiveArtifacts artifacts: 'test_results_oai_cn5g.html'
         }
       }
     }

ci-scripts/dsTestGenerateHTMLReport.py

@@ -34,9 +34,9 @@ class HtmlReport():
 
 	def generate(self):
 		cwd = os.getcwd()
-		self.file = open(cwd + '/ds_tester_results_oai_cn5g.html', 'w')
+		self.file = open(cwd + '/test_results_oai_cn5g.html', 'w')
 		self.generateHeader()
-
+		self.deploymentSummaryHeader()
 		finalStatus = self.testSummaryHeader()
 		self.testSummaryDetails()
 		self.testSummaryFooter()
@@ -81,9 +81,138 @@ class HtmlReport():
 		self.file.write('</div></body>\n')
 		self.file.write('</html>\n')
 
+	def deploymentSummaryHeader(self):
+	    self.file.write('  <h2>Deployment Summary</h2>\n')
+	    cwd = os.getcwd()
+	    if os.path.isfile(cwd + '/archives/deployment_status.log'):
+	        cmd = 'egrep -c "DEPLOYMENT: OK" archives/deployment_status.log || true'
+	        status = False
+	        ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	        if ret.stdout is not None:
+	            if ret.stdout.strip() == '1':
+	                status = True
+	        if status:
+	            self.file.write('  <div class="alert alert-success">\n')
+	            self.file.write('    <strong>Successful Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	            self.file.write('  </div>\n')
+	        else:
+	            self.file.write('  <div class="alert alert-danger">\n')
+	            self.file.write('    <strong>Failed Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	            self.file.write('  </div>\n')
+	    else:
+	        self.file.write('  <div class="alert alert-warning">\n')
+	        self.file.write('    <strong>LogFile not available! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	        self.file.write('  </div>\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <button data-toggle="collapse" data-target="#deployment-details">More details on Deployment</button>\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <div id="deployment-details" class="collapse">\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <table class="table-bordered" width = "80%" align = "center" border = 1>\n')
+	    self.file.write('     <tr bgcolor = "#33CCFF" >\n')
+	    self.file.write('       <th>Container Name</th>\n')
+	    self.file.write('       <th>Used Image Tag</th>\n')
+	    self.file.write('       <th>Image Creation Date</th>\n')
+	    self.file.write('       <th>Used Image Size</th>\n')
+	    self.file.write('       <th>Configuration Status</th>\n')
+	    self.file.write('     </tr>\n')
+	    self.addImageRow('mysql')
+	    self.addImageRow('oai_nrf')
+	    self.addImageRow('oai_amf')
+	    self.addImageRow('oai_smf')
+	    self.addImageRow('oai_spgwu')
+	    self.file.write('  </table>\n')
+	    self.file.write('  </div>\n')
+
+	def addImageRow(self, imageInfoPrefix):
+	    cwd = os.getcwd()
+	    if imageInfoPrefix == 'oai_amf':
+	        containerName = 'oai-amf'
+	        tagPattern = 'OAI_AMF_TAG'
+	        statusPrefix = 'amf'
+	    if imageInfoPrefix == 'oai_smf':
+	        containerName = 'oai-smf'
+	        tagPattern = 'OAI_SMF_TAG'
+	        statusPrefix = 'smf'
+	    if imageInfoPrefix == 'oai_nrf':
+	        containerName = 'oai-nrf'
+	        tagPattern = 'OAI_NRF_TAG'
+	        statusPrefix = 'nrf'
+	    if imageInfoPrefix == 'oai_spgwu':
+	        containerName = 'oai-spgwu-tiny'
+	        tagPattern = 'OAI_SPGWU_TAG'
+	        statusPrefix = 'spgwu'
+	    if imageInfoPrefix == 'mysql':
+	        containerName = imageInfoPrefix
+	        tagPattern = 'N/A'
+	    if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'):
+	        usedTag = ''
+	        createDate = ''
+	        size = ''
+	        with open(cwd + '/archives/' + imageInfoPrefix + '_image_info.log') as imageLog:
+	            for line in imageLog:
+	                line = line.strip()
+	                result = re.search(tagPattern + ': (?P<tag>[a-zA-Z0-9\-\_:]+)', line)
+	                if result is not None:
+	                    usedTag = result.group('tag')
+	                result = re.search('Date = (?P<date>[a-zA-Z0-9\-\_:]+)', line)
+	                if result is not None:
+	                    createDate = result.group('date')
+	                result = re.search('Size = (?P<size>[0-9]+) bytes', line)
+	                if result is not None:
+	                    sizeInt = int(result.group('size'))
+	                    if sizeInt < 1000000:
+	                        sizeInt = int(sizeInt / 1000)
+	                        size = str(sizeInt) + ' kB'
+	                    else:
+	                        sizeInt = int(sizeInt / 1000000)
+	                        size = str(sizeInt) + ' MB'
+	        imageLog.close()
+	        configState = 'KO'
+	        cmd = 'egrep -c "STATUS: healthy" archives/' + statusPrefix + '_config.log || true'
+	        ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	        if ret.stdout is not None:
+	            if ret.stdout.strip() == '1':
+	                configState = 'OK'
+	        self.file.write('     <tr>\n')
+	        self.file.write('       <td>' + containerName + '</td>\n')
+	        self.file.write('       <td>' + usedTag + '</td>\n')
+	        self.file.write('       <td>' + createDate + '</td>\n')
+	        self.file.write('       <td>' + size + '</td>\n')
+	        if configState == 'OK':
+	            self.file.write('       <td bgcolor = "DarkGreen"><b><font color="white">' + configState + '</font></b></td>\n')
+	        else:
+	            self.file.write('       <td bgcolor = "Red"><b><font color="white">' + configState + '</font></b></td>\n')
+	        self.file.write('     </tr>\n')
+	    else:
+	        if imageInfoPrefix == 'mysql':
+	            self.file.write('     <tr>\n')
+	            self.file.write('       <td>' + containerName + '</td>\n')
+	            self.file.write('       <td>mysql:5.7</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td>449MB</td>\n')
+	            configState = 'KO'
+	            cmd = 'egrep -c "STATUS: healthy" archives/mysql_status.log || true'
+	            ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	            if ret.stdout is not None:
+	                if ret.stdout.strip() == '1':
+	                    configState = 'OK'
+	            if configState == 'OK':
+	                self.file.write('       <td bgcolor = "DarkGreen"><b><font color="white">OK</font></b></td>\n')
+	            else:
+	                self.file.write('       <td bgcolor = "Red"><b><font color="white">KO</font></b></td>\n')
+	            self.file.write('     </tr>\n')
+	        else:
+	            self.file.write('     <tr>\n')
+	            self.file.write('       <td>' + containerName + '</td>\n')
+	            self.file.write('       <td>UNKNOWN</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td bgcolor = "DarkOrange"><b><font color="white">UNKNOW</font></b></td>\n')
+	            self.file.write('     </tr>\n')
+
 	def testSummaryHeader(self):
 		self.file.write('  <h2>DS Tester Summary</h2>\n')
-
 		finalDsTesterSummaryFile = 'DS-TEST-RESULTS/status.txt'
 		cwd = os.getcwd()
 		if os.path.isfile(cwd + '/' + finalDsTesterSummaryFile):

ci-scripts/dsTesterDockerCompose/amf-healthy-check.sh

@@ -2,7 +2,8 @@
 
 STATUS=0
 NB_UNREPLACED_AT=`cat /openair-amf/etc/*.conf | grep -v contact@openairinterface.org | grep -c @ || true`
-if [ $NB_UNREPLACED_AT -ne 0 ]
+NOT_FOUND=`ldd /openair-amf/bin/oai_amf | grep -c "not found" || true`
+if [[ $NB_UNREPLACED_AT -ne 0 || $NOT_FOUND -ne 0 ]]
 then
   STATUS=-1
 fi

ci-scripts/dsTesterDockerCompose/docker-compose.tplt

@@ -5,14 +5,19 @@ services:
         image: mysql:5.7
         ports:
              - 3306
-        command: --init-file /tmp/oai_db.sql
+        command: --init-file /docker-entrypoint-initdb.d/oai_db.sql
         volumes:
-            - ../../component/oai-amf/build/scripts/oai_db.sql:/tmp/oai_db.sql:rw
+            - ../../component/oai-amf/build/scripts/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql:rw
         environment:
             - MYSQL_DATABASE=oai_db
             - MYSQL_USER=test
             - MYSQL_PASSWORD=test
             - MYSQL_ROOT_PASSWORD=linux
+        healthcheck:
+            test: "/usr/bin/mysql --user=root --password=linux -e 'show databases;'"
+            interval: 10s
+            timeout: 5s
+            retries: 5
         networks:
             cicd_public_net:
                 ipv4_address: 192.168.61.194

ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh

@@ -2,7 +2,8 @@
 
 STATUS=0
 NB_UNREPLACED_AT=`cat /openair-nrf/etc/*.conf | grep -v contact@openairinterface.org | grep -c @ || true`
-if [ $NB_UNREPLACED_AT -ne 0 ]
+NOT_FOUND=`ldd /openair-nrf/bin/oai_nrf | grep -c "not found" || true`
+if [[ $NB_UNREPLACED_AT -ne 0 || $NOT_FOUND -ne 0 ]]
 then
   STATUS=-1
 fi

ci-scripts/dsTesterDockerCompose/smf-healthy-check.sh

@@ -2,7 +2,8 @@
 
 STATUS=0
 NB_UNREPLACED_AT=`cat /openair-smf/etc/*.conf | grep -v contact@openairinterface.org | grep -c @ || true`
-if [ $NB_UNREPLACED_AT -ne 0 ]
+NOT_FOUND=`ldd /openair-smf/bin/oai_smf | grep -c "not found" || true`
+if [[ $NB_UNREPLACED_AT -ne 0 || $NOT_FOUND -ne 0 ]]
 then
   STATUS=-1
 fi

ci-scripts/dsTesterDockerCompose/upf-healthy-check.sh

@@ -2,7 +2,8 @@
 
 STATUS=0
 NB_UNREPLACED_AT=`cat /openair-spgwc/etc/*.conf | grep -v contact@openairinterface.org | grep -c @ || true`
-if [ $NB_UNREPLACED_AT -ne 0 ]
+NOT_FOUND=`ldd /openair-spgwu-tiny/bin/oai_spgwu | grep -c "not found" || true`
+if [[ $NB_UNREPLACED_AT -ne 0 || $NOT_FOUND -ne 0 ]]
 then    
   STATUS=-1
 fi