feat(ci): adding AUSF/UDM/UDR deployment in pipeline

  * delayed deployment of the mySql container
  * added new components in deployment summary
  * smarter end-of-deployment detection
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -41,11 +41,6 @@ def ds_tester_ci_resource = params.DsTester
 // When triggered by upstream, specify which tag to use
 def upstreamTagToUse = params.upstreamTagToUse
 
-// Location of the 2nd CN executor
-def new_host_flag = false
-def new_host = ""
-def new_host_user = ""
-
 // Location of the CN tester
 def dsT_host_flag = false
 def dsT_host = ""
@@ -70,6 +65,13 @@ def smfBranch = params.smfBranch
 def spgwuTag = params.spgwuTag
 def spgwuBranch = params.spgwuBranch
 
+def ausfTag = params.ausfTag
+def ausfBranch = params.ausfBranch
+def udmTag = params.udmTag
+def udmBranch = params.udmBranch
+def udrTag = params.udrTag
+def udrBranch = params.udrBranch
+
 //-------------------------------------------------------------------------------
 // Pipeline start
 pipeline {
@@ -91,19 +93,8 @@ pipeline {
           JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
           JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
 
-          if (params.Host_CN_CI_2nd_Server_Flag != null) {
-            new_host_flag = params.Host_CN_CI_2nd_Server_Flag
-            if (new_host_flag) {
-              new_host = params.Host_CN_CI_2nd_Server
-              new_host_user = params.Host_CN_CI_2nd_Server_Login
-              echo "1st Node   is ${NODE_NAME}"
-              echo "2nd Node   is ${new_host}"
-            } else {
-              echo "Node       is ${NODE_NAME}"
-            }
-          } else {
-            echo "Node       is ${NODE_NAME}"
-          }
+          echo "Node       is ${NODE_NAME}"
+
           if (params.DS_Tester_Server_Flag != null) {
             dsT_host_flag = params.DS_Tester_Server_Flag
             if (dsT_host_flag) {
@@ -180,13 +171,6 @@ pipeline {
             sh "zip -r -qq oai-cn5g-fed.zip .git"
             sh "mkdir -p archives DS-TEST-RESULTS"
             sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
-            if (new_host_flag) {
-              // Prepare the workspace in remote server
-              copyTo2ndServer('oai-cn5g-fed.zip', new_host_flag, new_host_user, new_host)
-              myShCmd('git clean -x -d -f > /dev/null 2>&1', new_host_flag, new_host_user, new_host)
-              myShCmd('mkdir -p archives DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-              myShCmd('./scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch, new_host_flag, new_host_user, new_host)
-            }
           }
           if (scmEvent) {
             sh "git clean -x -d -f > /dev/null 2>&1"
@@ -196,16 +180,6 @@ pipeline {
             sh "zip -r -qq oai-cn5g-fed.zip .git"
             sh "mkdir -p archives DS-TEST-RESULTS"
             sh './scripts/syncComponents.sh --nrf-branch develop --amf-branch develop --smf-branch develop --spgwu-tiny-branch develop'
-            if (new_host_flag) {
-              // Prepare the workspace in remote server
-              copyTo2ndServer('oai-cn5g-fed.zip', new_host_flag, new_host_user, new_host)
-              myShCmd('git clean -x -d -f > /dev/null 2>&1', new_host_flag, new_host_user, new_host)
-              if ("MERGE".equals(env.gitlabActionType)) {
-                myShCmd("./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}", new_host_flag, new_host_user, new_host)
-              }
-              myShCmd('mkdir -p archives DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
-              myShCmd('./scripts/syncComponents.sh --nrf-branch develop --amf-branch develop --smf-branch develop --spgwu-tiny-branch develop', new_host_flag, new_host_user, new_host)
-            }
           }
           if ((!upstreamEvent) && (!scmEvent)) {
             sh "git clean -x -d -f > /dev/null 2>&1"
@@ -241,6 +215,27 @@ pipeline {
           } catch (Exception e) {
             error "OAI SPGW-U-Tiny Image tag to test does not exist!"
           }
+          try {
+            sh 'echo "OAI_AUSF_TAG: oai-ausf:' + ausfTag +'" > archives/oai_ausf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-ausf:' + ausfTag + ' >> archives/oai_ausf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-ausf:' + ausfTag + ' >> archives/oai_ausf_image_info.log'
+          } catch (Exception e) {
+            error "OAI AUSF Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_UDM_TAG: oai-udm:' + udmTag +'" > archives/oai_udm_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-udm:' + udmTag + ' >> archives/oai_udm_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-udm:' + udmTag + ' >> archives/oai_udm_image_info.log'
+          } catch (Exception e) {
+            error "OAI UDM Image tag to test does not exist!"
+          }
+          try {
+            sh 'echo "OAI_UDR_TAG: oai-udr:' + udrTag +'" > archives/oai_udr_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-udr:' + udrTag + ' >> archives/oai_udr_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-udr:' + udrTag + ' >> archives/oai_udr_image_info.log'
+          } catch (Exception e) {
+            error "OAI UDR Image tag to test does not exist!"
+          }
         }
       }
     }
@@ -250,17 +245,27 @@ pipeline {
           echo '\u2705 \u001B[32mDeploy CN5G using Docker-Compose\u001B[0m'
           // Prepare all needed files for docker-compose
           // First put all correct tags to test
-          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" -e "s#AMF_IMAGE_TAG#' + amfTag + '#" -e "s#SMF_IMAGE_TAG#' + smfTag + '#" -e "s#SPGWU_IMAGE_TAG#' + spgwuTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
+          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" -e "s#AMF_IMAGE_TAG#' + amfTag + '#" -e "s#SMF_IMAGE_TAG#' + smfTag + '#" -e "s#SPGWU_IMAGE_TAG#' + spgwuTag + '#" -e "s#AUSF_IMAGE_TAG#' + ausfTag + '#" -e "s#UDM_IMAGE_TAG#' + udmTag + '#" -e "s#UDR_IMAGE_TAG#' + udrTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
           dir('ci-scripts/dsTesterDockerCompose') {
-            sh 'docker-compose up -d  > ../../archives/compose_5gcn_up.log 2>&1'
-            sh 'sleep 100'
-            // Do a check on number of healthy containers
-            // 5 == mysql + nrf + amf + smf + upf(spgwu-tiny)
-            ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
-            ret = ret.trim()
-            if (ret != '5') {
+            // Tricking because mySql container takes a long time to be healthy
+            sh 'docker-compose up -d  cicd_mysql > ../../archives/compose_5gcn_up.log 2>&1'
+            sh 'sleep 30'
+            sh 'docker-compose up -d  >> ../../archives/compose_5gcn_up.log 2>&1'
+            int count = 0
+            while (count < 10) {
+              sh 'sleep 10'
+              // Do a check on number of healthy containers
+              // 8 == mysql + nrf + amf + smf + upf(spgwu-tiny) + ausf + udm + udr
+              ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
+              ret = ret.trim()
+              if (ret == '8') {
+                count = 20
+              }
+              count++
+            }
+            if (count < 20) {
               error "Deployment went wrong!"
-            }          
+            }
           }
         }
       }
@@ -276,29 +281,40 @@ pipeline {
             }
             // Do docker logs to recover the configuration results
             try {
-              //sh 'docker logs cicd-oai-nrf > archives/nrf_config.log 2>&1'
               sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf >> archives/nrf_config.log'
             } catch (Exception e) {
               sh 'echo "STATUS: KO" >> archives/nrf_config.log'
             }
             try {
-              //sh 'docker logs cicd-oai-amf > archives/amf_config.log 2>&1'
               sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-amf >> archives/amf_config.log'
             } catch (Exception e) {
               sh 'echo "STATUS: KO" >> archives/amf_config.log'
             }
             try {
-              //sh 'docker logs cicd-oai-smf > archives/smf_config.log 2>&1'
               sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-smf >> archives/smf_config.log'
             } catch (Exception e) {
               sh 'echo "STATUS: OK" >> archives/smf_config.log'
             }
             try {
-              //sh 'docker logs cicd-oai-upf > archives/spgwu_config.log 2>&1'
               sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-upf >> archives/spgwu_config.log'
             } catch (Exception e) {
               sh 'echo "STATUS: KO" >> archives/spgwu_config.log'
             }
+            try {
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-ausf >> archives/ausf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: OK" >> archives/ausf_config.log'
+            }
+            try {
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-udm >> archives/udm_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: OK" >> archives/udm_config.log'
+            }
+            try {
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-udr >> archives/udr_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: OK" >> archives/udr_config.log'
+            }
           }
         }
         success {
@@ -350,18 +366,20 @@ pipeline {
           script {
             // Copy the pcaps and logs from the containers
             sh "mkdir -p archives/pcaps archives/logs"
-            try {
-              sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap'
-              sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log'
-              sh 'docker cp cicd-oai-amf:/tmp/amf.pcap archives/pcaps/oai_amf.pcap'
-              sh 'docker cp cicd-oai-amf:/tmp/amf.log archives/logs/oai_amf.log'
-              sh 'docker cp cicd-oai-smf:/tmp/smf.pcap archives/pcaps/oai_smf.pcap'
-              sh 'docker cp cicd-oai-smf:/tmp/smf.log archives/logs/oai_smf.log'
-              sh 'docker cp cicd-oai-upf:/tmp/spgwu.pcap archives/pcaps/oai_spgwu.pcap'
-              sh 'docker cp cicd-oai-upf:/tmp/spgwu.log archives/logs/oai_spgwu.log'
-            } catch (Exception e) {
-              sh 'echo "Error in copying pcaps & logs from the containers"'
-            }
+            sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap || true'
+            sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log || true'
+            sh 'docker cp cicd-oai-amf:/tmp/amf.pcap archives/pcaps/oai_amf.pcap || true'
+            sh 'docker cp cicd-oai-amf:/tmp/amf.log archives/logs/oai_amf.log || true'
+            sh 'docker cp cicd-oai-smf:/tmp/smf.pcap archives/pcaps/oai_smf.pcap || true'
+            sh 'docker cp cicd-oai-smf:/tmp/smf.log archives/logs/oai_smf.log || true'
+            sh 'docker cp cicd-oai-upf:/tmp/spgwu.pcap archives/pcaps/oai_spgwu.pcap || true'
+            sh 'docker cp cicd-oai-upf:/tmp/spgwu.log archives/logs/oai_spgwu.log || true'
+            sh 'docker cp cicd-oai-ausf:/tmp/ausf.pcap archives/pcaps/oai_ausf.pcap || true'
+            sh 'docker cp cicd-oai-ausf:/tmp/ausf.log archives/logs/oai_ausf.log || true'
+            sh 'docker cp cicd-oai-udm:/tmp/udm.pcap archives/pcaps/oai_udm.pcap || true'
+            sh 'docker cp cicd-oai-udm:/tmp/udm.log archives/logs/oai_udm.log || true'
+            sh 'docker cp cicd-oai-udr:/tmp/udr.pcap archives/pcaps/oai_udr.pcap || true'
+            sh 'docker cp cicd-oai-udr:/tmp/udr.log archives/logs/oai_udr.log || true'
           }
         }
       }
@@ -383,11 +401,14 @@ pipeline {
       script {
         // Get logs if deployment fail
         if (deployed != true) {
-        sh "mkdir -p archives/logs"
-        sh 'docker logs cicd-oai-nrf > archives/logs/oai_nrf.log'
-        sh 'docker logs cicd-oai-amf > archives/logs/oai_amf.log'
-        sh 'docker logs cicd-oai-smf > archives/logs/oai_smf.log'
-        sh 'docker logs cicd-oai-upf > archives/logs/oai_spgwu.log'
+          sh "mkdir -p archives/logs"
+          sh 'docker logs cicd-oai-nrf > archives/logs/oai_nrf.log'
+          sh 'docker logs cicd-oai-amf > archives/logs/oai_amf.log'
+          sh 'docker logs cicd-oai-smf > archives/logs/oai_smf.log'
+          sh 'docker logs cicd-oai-upf > archives/logs/oai_spgwu.log'
+          sh 'docker logs cicd-oai-ausf > archives/logs/oai_ausf.log'
+          sh 'docker logs cicd-oai-udm > archives/logs/oai_udm.log'
+          sh 'docker logs cicd-oai-udr > archives/logs/oai_udr.log'
         } 
         // Remove any leftover containers/networks
         sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host

ci-scripts/dsTestGenerateHTMLReport2.py

@@ -122,6 +122,9 @@ class HtmlReport():
 		self.addImageRow('oai_amf')
 		self.addImageRow('oai_smf')
 		self.addImageRow('oai_spgwu')
+		self.addImageRow('oai_ausf')
+		self.addImageRow('oai_udm')
+		self.addImageRow('oai_udr')
 		self.file.write('  </table>\n')
 		self.file.write('  </div>\n')
 
@@ -143,6 +146,18 @@ class HtmlReport():
 			containerName = 'oai-spgwu-tiny'
 			tagPattern = 'OAI_SPGWU_TAG'
 			statusPrefix = 'spgwu'
+		if imageInfoPrefix == 'oai_ausf':
+			containerName = 'oai-ausf'
+			tagPattern = 'OAI_AUSF_TAG'
+			statusPrefix = 'ausf'
+		if imageInfoPrefix == 'oai_udm':
+			containerName = 'oai-udm'
+			tagPattern = 'OAI_UDM_TAG'
+			statusPrefix = 'udm'
+		if imageInfoPrefix == 'oai_udr':
+			containerName = 'oai-udr'
+			tagPattern = 'OAI_UDR_TAG'
+			statusPrefix = 'udr'
 		if imageInfoPrefix == 'mysql':
 			containerName = imageInfoPrefix
 			tagPattern = 'N/A'
@@ -252,7 +267,7 @@ class HtmlReport():
 		cwd = os.getcwd()
 		if os.path.isfile(cwd + '/DS-TEST-RESULTS/mvc.yaml'):
 			with open(cwd + '/DS-TEST-RESULTS/mvc.yaml') as f:
-				data = yaml.load(f)
+				data = yaml.full_load(f)
 				nScenarios = len(data['scenarios'])
 				for scenario in range(nScenarios):
 					self.file.write('     <tr>\n')

ci-scripts/toCheckDSTesterResult1.py

@@ -44,7 +44,7 @@ except IOError:
 if locexist:
     try:
         with open(cwd + f'/DS-TEST-RESULTS/{filename.group(0)}') as f:
-            data = yaml.load(f)
+            data = yaml.full_load(f)
             if data["final-result"] == 'fail':
                 sys.exit('DsTester final result FAILED')
     except IOError: