Demo with dsTest

- demo analysis and reproduction steps

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "component/oai-upf-equivalent"]
 	path = component/oai-upf-equivalent
 	url = https://github.com/OPENAIRINTERFACE/openair-spgwu-tiny.git
+[submodule "component/oai-nrf"]
+	path = component/oai-nrf
+	url = https://gitlab.eurecom.fr/oai/cn5g/oai-cn5g-nrf.git

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -38,6 +38,9 @@ def cn_ci_host = params.Host_CN_CI_Server
 def cn_ci_resource = params.DockerContainers
 def ds_tester_ci_resource = params.DsTester
 
+// Location of the DsTester workspace
+def dsTestFrameworkLocation = params.dsTestFrameworkLocation
+
 // When triggered by upstream, specify which tag to use
 def upstreamTagToUse = params.upstreamTagToUse
 
@@ -57,12 +60,14 @@ def scmEvent = false
 def upstreamEvent = false
 
 // Default tags / branches  --> could be passed on by upstream job or by PR content
-def amfTag = 'develop'
-def amfBranch = 'develop'
-def smfTag = 'develop'
-def smfBranch = 'develop'
-def spgwuTag = 'develop'
-def spgwuBranch = 'develop'
+def nrfTag = params.nrfTag
+def nrfBranch = params.nrfBranch
+def amfTag = params.amfTag
+def amfBranch = params.amfBranch
+def smfTag = params.smfTag
+def smfBranch = params.smfBranch
+def spgwuTag = params.spgwuTag
+def spgwuBranch = params.spgwuBranch
 
 //-------------------------------------------------------------------------------
 // Pipeline start
@@ -130,12 +135,20 @@ pipeline {
           for (cause in currentBuild.getBuildCauses()) {
             if (cause.toString() ==~ /.*UpstreamCause.*/) {
               upstreamEvent = true
-            } else {
-              scmEvent = true
+            //} else {
+            //  scmEvent = true
             }
           }
 
           if (upstreamEvent) {
+            if (params.NRF_TAG != null) {
+              nrfTag = params.NRF_TAG
+              echo "Upstream Job passed NRF_TAG to use: ${nrfTag}"
+            }
+            if (params.NRF_BRANCH != null) {
+              nrfBranch = params.NRF_BRANCH
+              echo "Upstream Job passed NRF_BRANCH to use: ${nrfBranch}"
+            }
             if (params.AMF_TAG != null) {
               amfTag = params.AMF_TAG
               echo "Upstream Job passed AMF_TAG to use: ${amfTag}"
@@ -157,13 +170,13 @@ pipeline {
             sh 'git checkout -f ' + upstreamTagToUse
             sh "zip -r -qq oai-cn5g-fed.zip .git"
             sh "mkdir -p archives DS-TEST-RESULTS"
-            sh './scripts/syncComponents.sh --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch
+            sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
             if (new_host_flag) {
               // Prepare the workspace in remote server
               copyTo2ndServer('oai-cn5g-fed.zip', new_host_flag, new_host_user, new_host)
               myShCmd('git clean -x -d -f > /dev/null 2>&1', new_host_flag, new_host_user, new_host)
               myShCmd('mkdir -p archives DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-              myShCmd('./scripts/syncComponents.sh --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch, new_host_flag, new_host_user, new_host)
+              myShCmd('./scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch, new_host_flag, new_host_user, new_host)
             }
           }
           if (scmEvent) {
@@ -187,41 +200,112 @@ pipeline {
           }
           if ((!upstreamEvent) && (!scmEvent)) {
             sh "git clean -x -d -f > /dev/null 2>&1"
-            sh './scripts/syncComponents.sh --hss-branch ' + hssBranch + ' --mme-branch ' + mmeBranch + ' --spgwc-branch ' + spgwcBranch + ' --spgwu-tiny-branch ' + spgwuBranch
+            sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
             sh "mkdir -p archives DS-TEST-RESULTS"
           }
+          // Verify that the images are available
+          try {
+            sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+          } catch (Exception e) {
+            error "OAI NRF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_AMF_TAG: oai-amf:' + amfTag +'" > archives/oai_amf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
+          } catch (Exception e) {
+            error "OAI AMF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_SMF_TAG: oai-smf:' + smfTag +'" > archives/oai_smf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
+          } catch (Exception e) {
+            error "OAI SMF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_SPGWU_TAG: oai-spgwu-tiny:' + spgwuTag +'" > archives/oai_spgwu_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
+          } catch (Exception e) {
+            error "OAI SPGW-U-Tiny Image tag to test does not exist!"
+          }
         }
       }
     }
-    stage ('Deploy Whole EPC') {
-      stages {
-        stage ('Create Public Network') {
-          steps {
-            script {
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=CreateNetworks', new_host_flag, new_host_user, new_host)
-            }
+    stage ('Deploy Whole 5G Core Network') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mDeploy CN5G in idle mode\u001B[0m'
+          // Prepare all needed files for docker-compose
+          // First put all correct tags to test
+          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" -e "s#AMF_IMAGE_TAG#' + amfTag + '#" -e "s#SMF_IMAGE_TAG#' + smfTag + '#" -e "s#SPGWU_IMAGE_TAG#' + spgwuTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
+          // Entrypoints are modified to be inactive for dsTester framework
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-nrf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/nrf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-amf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/amf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-smf/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/smf-entrypoint-sleep.sh'
+          sh 'sed -e "s@exec.*@sleep infinity@" component/oai-upf-equivalent/scripts/entrypoint.sh > ci-scripts/dsTesterDockerCompose/upf-entrypoint-sleep.sh'
+          sh 'chmod 775 ci-scripts/dsTesterDockerCompose/*entrypoint-sleep.sh'
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose up -d  > ../../archives/compose_5gcn_up.log 2>&1'
+            sh 'sleep 100'
+            // Do a check on number of healthy containers
+            // 5 == mysql + nrf + amf + smf + upf(spgwu-tiny)
+            ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
+            ret = ret.trim()
+            if (ret != '5') {
+              error "Deployment went wrong!"
+            }          
           }
         }
-        stage ('Deploy Containers') {
-          steps {
-            script {
-              // Deploy and configure MySQL Server
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployMySqlServer', new_host_flag, new_host_user, new_host)
-              // Deploy and configure AMF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployAMF --tag=' + amfTag, new_host_flag, new_host_user, new_host)
-              // Deploy and configure SMF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeploySMF --tag=' + smfTag, new_host_flag, new_host_user, new_host)
-              // Deploy and configure UPF
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=DeployUPF --tag=develop', new_host_flag, new_host_user, new_host)
+      }
+      post {
+        always {
+          script {
+            // Check status on mysql.
+            try {
+              sh 'docker exec cicd-mysql-svr /bin/bash -c "mysqladmin -u root --password=linux ping" > archives/mysql_status.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-mysql-svr >> archives/mysql_status.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/mysql_status.log'
             }
-          }
-          post {
-            always {
-              script {
-                copyFrom2ndServer('archives/*_config.log', 'archives', new_host_flag, new_host_user, new_host)
-                sh 'python3 ./ci-scripts/generateHtmlReportDeployment.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
-              }
+            // Do docker logs to recover the configuration results
+            try {
+              sh 'docker logs cicd-oai-nrf > archives/nrf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf >> archives/nrf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/nrf_config.log'
+            }
+            try {
+              sh 'docker logs cicd-oai-amf > archives/amf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-amf >> archives/amf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/amf_config.log'
+            }
+            try {
+              sh 'docker logs cicd-oai-smf > archives/smf_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-smf >> archives/smf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: OK" >> archives/smf_config.log'
             }
+            try {
+              sh 'docker logs cicd-oai-upf > archives/spgwu_config.log 2>&1'
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-upf >> archives/spgwu_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/spgwu_config.log'
+            }
+          }
+        }
+        success {
+          script {
+            sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log'
+          }
+        }
+        unsuccessful {
+          script {
+            sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log'
           }
         }
       }
@@ -231,32 +315,21 @@ pipeline {
       steps {
         lock (ds_tester_ci_resource) {
           script {
-//            myShCmd('cd /home/oaici/CI-dev-ops-cn5g && git clean -x -d -f > /dev/null', new_host_flag, new_host_user, new_host)
-//            myShCmdWithLog('cd /home/oaici/CI-dev-ops-cn5g/scripts && CI_ENV=True SRC_BASE_DIR=/tmp/CI-CN5G-FED ./run-5gc.bash -pt --shark --detach --get-results --5g', 'archives/run-5g-dstester.log', new_host_flag, new_host_user, new_host)
-//            copyFrom2ndServer('DS-TEST-RESULTS/status.txt', 'DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-//            copyFrom2ndServer('DS-TEST-RESULTS/*.tar', 'DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-            sh 'cd /home/oaici/CI-dev-ops-cn5g && git clean -x -d -f > /dev/null'
-            sh 'cd /home/oaici/CI-dev-ops-cn5g/scripts && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && CI_ENV=True SRC_BASE_DIR=' + WORKSPACE + ' ./run-5gc.bash -pt --shark --detach --get-results --5g > ' + WORKSPACE + '/archives/run-5g-dstester.log 2>&1'
-            sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
+            sh 'cd ' + dsTestFrameworkLocation + ' && git clean -x -d -f > /dev/null'
+            sh 'cd ' + dsTestFrameworkLocation + '/scripts && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && CI_ENV=True SRC_BASE_DIR=' + WORKSPACE + ' ./run-5gc.bash -pt --shark --detach --get-results --5g --pcap-test > ' + WORKSPACE + '/archives/run-5g-dstester.log 2>&1'
+            sh 'cd ' + dsTestFrameworkLocation + ' && git stash > /dev/null'
+            sh 'cd ' + dsTestFrameworkLocation + ' && git stash clear > /dev/null'
+            sh 'cd ' + WORKSPACE + ' && python3 ./ci-scripts/toCheckDSTesterResult.py' 		
           }
         }
       }
     }
-    stage ('Undeploy EPC') {
-      stages {
-        stage ('Undeploy Containers') {
-          steps {
-            script {
-              // Remove the containers
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveAllContainers', new_host_flag, new_host_user, new_host)
-            }
-          }
-        }
-        stage ('Delete Public Network') {
-          steps {
-            script {
-              myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveNetworks', new_host_flag, new_host_user, new_host)
-            }
+    stage ('Undeploy 5G-CN') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mUn-Deploy CN5G\u001B[0m'
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1'
           }
         }
       }
@@ -265,20 +338,20 @@ pipeline {
   post {
     always {
       script { 
-        // Removing all containers
-        myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveAllContainers', new_host_flag, new_host_user, new_host)
-        // Removing the network
-        myShCmd('python3 ./ci-scripts/dsTestDeployTools.py --action=RemoveNetworks', new_host_flag, new_host_user, new_host)
+        // Remove any leftover containers/networks
+        dir('ci-scripts/dsTesterDockerCompose') {
+          sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1'
+        }
+        // Generating the HTML report
+        sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
+
         // Zipping all archived log files
         sh "zip -r -qq cn5g_fed_docker_logs.zip archives DS-TEST-RESULTS/*.tar DS-TEST-RESULTS/status.txt"
         if (fileExists('cn5g_fed_docker_logs.zip')) {
           archiveArtifacts artifacts: 'cn5g_fed_docker_logs.zip'
         }
-        if (fileExists('deploy_results_oai_cn5g.html')) {
-          archiveArtifacts artifacts: 'deploy_results_oai_cn5g.html'
-        }
-        if (fileExists('ds_tester_results_oai_cn5g.html')) {
-          archiveArtifacts artifacts: 'ds_tester_results_oai_cn5g.html'
+        if (fileExists('test_results_oai_cn5g.html')) {
+          archiveArtifacts artifacts: 'test_results_oai_cn5g.html'
         }
       }
     }

ci-scripts/dsTestDeployTools.py

@@ -102,7 +102,7 @@ class deployForDsTester():
         # check if there is an entrypoint
         entrypoint = re.search('entrypoint', str(res))
         if entrypoint is not None:
-            print('not supported yet')
+            subprocess_run_w_echo('docker run --privileged --name cicd-oai-amf --network cicd-oai-public-net --ip ' + CICD_AMF_PUBLIC_ADDR + ' -d --entrypoint "/bin/bash" oai-amf:' + self.tag + ' -c "sleep infinity"')
         else:
             subprocess_run_w_echo('docker run --privileged --name cicd-oai-amf --network cicd-oai-public-net --ip ' + CICD_AMF_PUBLIC_ADDR + ' -d oai-amf:' + self.tag + ' /bin/bash -c "sleep infinity"')
         subprocess_run_w_echo('sed -e "s@CI_NGAP_IF_NAME@eth0@" -e "s@CI_N11_IF_NAME@eth0@" -e "s@CI_SMF0_IP_ADDRESS@' + CICD_SMF_PUBLIC_ADDR + '@" -e "s@CI_SMF1_IP_ADDRESS@' + CICD_DUMMY_SMF_PUBLIC_ADDR + '@" -e "s@CI_MYSQL_IP_ADDRESS@' + CICD_MYSQL_PUBLIC_ADDR + '@" ci-scripts/temp/generate_amf_conf.sh > ci-scripts/temp/ci-generate_amf_conf.sh')
@@ -120,7 +120,7 @@ class deployForDsTester():
         # check if there is an entrypoint
         entrypoint = re.search('entrypoint', str(res))
         if entrypoint is not None:
-            print('not supported yet')
+            subprocess_run_w_echo('docker run --privileged --name cicd-oai-smf --network cicd-oai-public-net --ip ' + CICD_SMF_PUBLIC_ADDR + ' -d --entrypoint "/bin/bash" oai-smf:' + self.tag + ' -c "sleep infinity"')
         else:
             subprocess_run_w_echo('docker run --privileged --name cicd-oai-smf --network cicd-oai-public-net --ip ' + CICD_SMF_PUBLIC_ADDR + ' -d oai-smf:' + self.tag + ' /bin/bash -c "sleep infinity"')
         subprocess_run_w_echo('sed -e "s@CI_N4_IF_NAME@eth0@" -e "s@CI_SBI_IF_NAME@eth0@" -e "s@CI_AMF_IP_ADDR@' + CICD_AMF_PUBLIC_ADDR + '@" -e "s@CI_UPF_IP_ADDR@' + CICD_UPF_PUBLIC_ADDR + '@" ci-scripts/temp/generate_smf_conf.sh > ci-scripts/temp/ci-generate_smf_conf.sh')

ci-scripts/dsTestGenerateHTMLReport.py

@@ -34,9 +34,9 @@ class HtmlReport():
 
 	def generate(self):
 		cwd = os.getcwd()
-		self.file = open(cwd + '/ds_tester_results_oai_cn5g.html', 'w')
+		self.file = open(cwd + '/test_results_oai_cn5g.html', 'w')
 		self.generateHeader()
-
+		self.deploymentSummaryHeader()
 		finalStatus = self.testSummaryHeader()
 		self.testSummaryDetails()
 		self.testSummaryFooter()
@@ -47,7 +47,7 @@ class HtmlReport():
 		if finalStatus:
 			sys.exit(0)
 		else:
-			sys.exit(-1)
+			print("DS-TESTER testing FAILED")
 
 	def generateHeader(self):
 		# HTML Header
@@ -81,9 +81,138 @@ class HtmlReport():
 		self.file.write('</div></body>\n')
 		self.file.write('</html>\n')
 
+	def deploymentSummaryHeader(self):
+	    self.file.write('  <h2>Deployment Summary</h2>\n')
+	    cwd = os.getcwd()
+	    if os.path.isfile(cwd + '/archives/deployment_status.log'):
+	        cmd = 'egrep -c "DEPLOYMENT: OK" archives/deployment_status.log || true'
+	        status = False
+	        ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	        if ret.stdout is not None:
+	            if ret.stdout.strip() == '1':
+	                status = True
+	        if status:
+	            self.file.write('  <div class="alert alert-success">\n')
+	            self.file.write('    <strong>Successful Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	            self.file.write('  </div>\n')
+	        else:
+	            self.file.write('  <div class="alert alert-danger">\n')
+	            self.file.write('    <strong>Failed Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	            self.file.write('  </div>\n')
+	    else:
+	        self.file.write('  <div class="alert alert-warning">\n')
+	        self.file.write('    <strong>LogFile not available! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
+	        self.file.write('  </div>\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <button data-toggle="collapse" data-target="#deployment-details">More details on Deployment</button>\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <div id="deployment-details" class="collapse">\n')
+	    self.file.write('  <br>\n')
+	    self.file.write('  <table class="table-bordered" width = "80%" align = "center" border = 1>\n')
+	    self.file.write('     <tr bgcolor = "#33CCFF" >\n')
+	    self.file.write('       <th>Container Name</th>\n')
+	    self.file.write('       <th>Used Image Tag</th>\n')
+	    self.file.write('       <th>Image Creation Date</th>\n')
+	    self.file.write('       <th>Used Image Size</th>\n')
+	    self.file.write('       <th>Configuration Status</th>\n')
+	    self.file.write('     </tr>\n')
+	    self.addImageRow('mysql')
+	    self.addImageRow('oai_nrf')
+	    self.addImageRow('oai_amf')
+	    self.addImageRow('oai_smf')
+	    self.addImageRow('oai_spgwu')
+	    self.file.write('  </table>\n')
+	    self.file.write('  </div>\n')
+
+	def addImageRow(self, imageInfoPrefix):
+	    cwd = os.getcwd()
+	    if imageInfoPrefix == 'oai_amf':
+	        containerName = 'oai-amf'
+	        tagPattern = 'OAI_AMF_TAG'
+	        statusPrefix = 'amf'
+	    if imageInfoPrefix == 'oai_smf':
+	        containerName = 'oai-smf'
+	        tagPattern = 'OAI_SMF_TAG'
+	        statusPrefix = 'smf'
+	    if imageInfoPrefix == 'oai_nrf':
+	        containerName = 'oai-nrf'
+	        tagPattern = 'OAI_NRF_TAG'
+	        statusPrefix = 'nrf'
+	    if imageInfoPrefix == 'oai_spgwu':
+	        containerName = 'oai-spgwu-tiny'
+	        tagPattern = 'OAI_SPGWU_TAG'
+	        statusPrefix = 'spgwu'
+	    if imageInfoPrefix == 'mysql':
+	        containerName = imageInfoPrefix
+	        tagPattern = 'N/A'
+	    if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'):
+	        usedTag = ''
+	        createDate = ''
+	        size = ''
+	        with open(cwd + '/archives/' + imageInfoPrefix + '_image_info.log') as imageLog:
+	            for line in imageLog:
+	                line = line.strip()
+	                result = re.search(tagPattern + ': (?P<tag>[a-zA-Z0-9\-\_:]+)', line)
+	                if result is not None:
+	                    usedTag = result.group('tag')
+	                result = re.search('Date = (?P<date>[a-zA-Z0-9\-\_:]+)', line)
+	                if result is not None:
+	                    createDate = result.group('date')
+	                result = re.search('Size = (?P<size>[0-9]+) bytes', line)
+	                if result is not None:
+	                    sizeInt = int(result.group('size'))
+	                    if sizeInt < 1000000:
+	                        sizeInt = int(sizeInt / 1000)
+	                        size = str(sizeInt) + ' kB'
+	                    else:
+	                        sizeInt = int(sizeInt / 1000000)
+	                        size = str(sizeInt) + ' MB'
+	        imageLog.close()
+	        configState = 'KO'
+	        cmd = 'egrep -c "STATUS: healthy" archives/' + statusPrefix + '_config.log || true'
+	        ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	        if ret.stdout is not None:
+	            if ret.stdout.strip() == '1':
+	                configState = 'OK'
+	        self.file.write('     <tr>\n')
+	        self.file.write('       <td>' + containerName + '</td>\n')
+	        self.file.write('       <td>' + usedTag + '</td>\n')
+	        self.file.write('       <td>' + createDate + '</td>\n')
+	        self.file.write('       <td>' + size + '</td>\n')
+	        if configState == 'OK':
+	            self.file.write('       <td bgcolor = "DarkGreen"><b><font color="white">' + configState + '</font></b></td>\n')
+	        else:
+	            self.file.write('       <td bgcolor = "Red"><b><font color="white">' + configState + '</font></b></td>\n')
+	        self.file.write('     </tr>\n')
+	    else:
+	        if imageInfoPrefix == 'mysql':
+	            self.file.write('     <tr>\n')
+	            self.file.write('       <td>' + containerName + '</td>\n')
+	            self.file.write('       <td>mysql:5.7</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td>449MB</td>\n')
+	            configState = 'KO'
+	            cmd = 'egrep -c "STATUS: healthy" archives/mysql_status.log || true'
+	            ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+	            if ret.stdout is not None:
+	                if ret.stdout.strip() == '1':
+	                    configState = 'OK'
+	            if configState == 'OK':
+	                self.file.write('       <td bgcolor = "DarkGreen"><b><font color="white">OK</font></b></td>\n')
+	            else:
+	                self.file.write('       <td bgcolor = "Red"><b><font color="white">KO</font></b></td>\n')
+	            self.file.write('     </tr>\n')
+	        else:
+	            self.file.write('     <tr>\n')
+	            self.file.write('       <td>' + containerName + '</td>\n')
+	            self.file.write('       <td>UNKNOWN</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td>N/A</td>\n')
+	            self.file.write('       <td bgcolor = "DarkOrange"><b><font color="white">UNKNOW</font></b></td>\n')
+	            self.file.write('     </tr>\n')
+
 	def testSummaryHeader(self):
 		self.file.write('  <h2>DS Tester Summary</h2>\n')
-
 		finalDsTesterSummaryFile = 'DS-TEST-RESULTS/status.txt'
 		cwd = os.getcwd()
 		if os.path.isfile(cwd + '/' + finalDsTesterSummaryFile):

ci-scripts/dsTesterDockerCompose/amf-healthy-check.sh

+#!/bin/bash
+
+STATUS=0
+NB_UNREPLACED_AT=`cat /openair-amf/etc/*.conf | grep -v contact@openairinterface.org | grep -c @ || true`
+NOT_FOUND=`ldd /openair-amf/bin/oai_amf | grep -c "not found" || true`
+if [[ $NB_UNREPLACED_AT -ne 0 || $NOT_FOUND -ne 0 ]]
+then
+  STATUS=-1
+fi
+
+exit $STATUS

ci-scripts/dsTesterDockerCompose/docker-compose.tplt

+version: '3.8'
+services:
+    cicd_mysql:
+        container_name: cicd-mysql-svr
+        image: mysql:5.7
+        ports:
+             - 3306
+        command: --init-file /docker-entrypoint-initdb.d/oai_db.sql
+        volumes:
+            - ../../docker-compose/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql:rw
+        environment:
+            - TZ=Europe/Paris
+            - MYSQL_DATABASE=oai_db
+            - MYSQL_USER=test
+            - MYSQL_PASSWORD=test
+            - MYSQL_ROOT_PASSWORD=linux
+        healthcheck:
+            test: "/usr/bin/mysql --user=root --password=linux -e 'show databases;'"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+        networks:
+            cicd_public_net:
+                ipv4_address: 192.168.61.194
+
+    cicd_oai_nrf:
+        container_name: cicd-oai-nrf
+        image: oai-nrf:NRF_IMAGE_TAG
+        ports:
+            - 80
+            - 9090
+        environment:
+            - TZ=Europe/Paris
+            - NRF_INTERFACE_NAME_FOR_SBI=eth0
+            - NRF_INTERFACE_PORT_FOR_SBI=80
+            - NRF_INTERFACE_HTTP2_PORT_FOR_SBI=9090
+            - NRF_API_VERSION=v1
+            - INSTANCE=0
+            - PID_DIRECTORY=/var/run
+        networks:
+            cicd_public_net:
+                ipv4_address: 192.168.61.195
+        volumes:
+            - ./nrf-entrypoint-sleep.sh:/openair-nrf/bin/nrf-entrypoint-sleep.sh
+            - ./nrf-healthy-check.sh:/openair-nrf/bin/nrf-healthy-check.sh
+        entrypoint: /bin/bash -c "/openair-nrf/bin/nrf-entrypoint-sleep.sh"
+        healthcheck:
+            test: /bin/bash -c "/openair-nrf/bin/nrf-healthy-check.sh"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+    cicd_oai_amf:
+        container_name: cicd-oai-amf
+        image: oai-amf:AMF_IMAGE_TAG
+        ports:
+             - 38412
+             - 80
+        environment:
+            - INSTANCE=1
+            - PID_DIRECTORY=/var/run
+            - MCC=208
+            - MNC=95
+            - REGION_ID=128
+            - AMF_SET_ID=1
+            - SERVED_GUAMI_MCC_0=208
+            - SERVED_GUAMI_MNC_0=95
+            - SERVED_GUAMI_REGION_ID_0=128
+            - SERVED_GUAMI_AMF_SET_ID_0=1
+            - SERVED_GUAMI_MCC_1=460
+            - SERVED_GUAMI_MNC_1=11
+            - SERVED_GUAMI_REGION_ID_1=10
+            - SERVED_GUAMI_AMF_SET_ID_1=1
+            - PLMN_SUPPORT_MCC=208
+            - PLMN_SUPPORT_MNC=95
+            - PLMN_SUPPORT_TAC=0xa000
+            - SST_0=222
+            - SD_0=123
+            - SST_1=1
+            - SD_1=12
+            - AMF_INTERFACE_NAME_FOR_NGAP=eth0
+            - AMF_INTERFACE_NAME_FOR_N11=eth0
+            - SMF_INSTANCE_ID_0=1
+            - SMF_IPV4_ADDR_0=192.168.61.197
+            - SMF_HTTP_VERSION_0=v1
+            - SELECTED_0=true
+            - SMF_INSTANCE_ID_1=2
+            - SMF_IPV4_ADDR_1=192.168.61.197
+            - SMF_HTTP_VERSION_1=v1
+            - SELECTED_1=false
+            - MYSQL_SERVER=cicd-mysql-svr
+            - MYSQL_USER=root
+            - MYSQL_PASS=linux
+            - MYSQL_DB=oai_db
+            - OPERATOR_KEY=63bfa50ee6523365ff14c1f45f88737d
+            - NRF_IPV4_ADDRESS=192.168.61.195
+            - NRF_PORT=80
+            - NF_REGISTRATION=yes
+            - SMF_SELECTION=yes
+            - NRF_API_VERSION=v1
+            - AUSF_IPV4_ADDRESS=192.168.61.210
+            - AUSF_PORT=80
+            - AUSF_API_VERSION=v1
+        depends_on:
+            - cicd_mysql
+            - cicd_oai_nrf
+        networks:
+            cicd_public_net:
+                ipv4_address: 192.168.61.196
+        volumes:
+            - ./amf-entrypoint-sleep.sh:/openair-amf/bin/amf-entrypoint-sleep.sh
+            - ./amf-healthy-check.sh:/openair-amf/bin/amf-healthy-check.sh
+        entrypoint: /bin/bash -c "/openair-amf/bin/amf-entrypoint-sleep.sh"
+        healthcheck:
+            test: /bin/bash -c "/openair-amf/bin/amf-healthy-check.sh"
+            interval: 10s