chore(ci): some tests are mandatory to pass for ngap-tester

* tracing PCAP file per test-case

ci-scripts/Jenkinsfile-GitLab-NGAP-Tester

@@ -205,12 +205,6 @@ pipeline {
           sh 'cp -Rf /opt/ngap-tester/test/* ci-scripts/docker-compose/ngap-tester/'
           updateDockerCompose('ci-scripts/docker-compose/ngap-tester/docker-compose-basic-nrf.yaml', ngapTesterTag)
           dir ('ci-scripts/docker-compose/ngap-tester') {
-            sh 'docker-compose -f docker-compose-basic-nrf.yaml up -d mysql'
-            sleep 2
-            sh 'sudo rm -f /tmp/oai-cn5g-ngap-tester.*'
-            sh 'nohup sudo tshark -i demo-oai -f "sctp or port 80 or port 8080 or port 8805 or icmp or port 3306 or port 2152" -w /tmp/oai-cn5g-ngap-tester.pcap > /tmp/oai-cn5g-ngap-tester.log 2>&1 &'
-            sh '../../checkTsharkCapture.py --log_file /tmp/oai-cn5g-ngap-tester.log --timeout 30'
-            sh 'sudo chmod 666 /tmp/oai-cn5g-ngap-tester.*'
             sh 'docker-compose -f docker-compose-basic-nrf.yaml up -d'
             sh '../../checkContainerStatus.py --container_name mysql --timeout 40'
             sh '../../checkContainerStatus.py --container_name oai-spgwu --timeout 40'
@@ -247,12 +241,20 @@ pipeline {
           for (testCase in testScenarios) {
             echo "Test-Case is ${testCase}"
             timeout(4) {
+              sh 'sudo rm -f /tmp/oai-cn5g-ngap-tester.*'
+              sh 'nohup sudo tshark -i demo-oai -f "sctp or port 80 or port 8080 or port 8805 or icmp or port 3306 or port 2152" -w /tmp/oai-cn5g-ngap-tester.pcap > /tmp/oai-cn5g-ngap-tester.log 2>&1 &'
+              sh './ci-scripts/checkTsharkCapture.py --log_file /tmp/oai-cn5g-ngap-tester.log --timeout 30'
+              sh 'sudo chmod 666 /tmp/oai-cn5g-ngap-tester.*'
               try {
                 sh "docker exec ngap-tester /bin/sh -c './ngap-tester run -o ${testCase}' > archives/${testCase}.log 2>&1"
                 echo "Test-Case #${testCase} seems to have COMPLETED (FAILED OR PASSED)!"
               } catch (Exception e) {
                 echo "Test-Case #${testCase} seems to have TIME-OUT or FAILED!"
               }
+              sleep 2
+              sh 'sudo killall -9 tshark'
+              sleep 1
+              sh "cp /tmp/oai-cn5g-ngap-tester.pcap archives/${testCase}.pcap"
             }
           }
           dir ('ci-scripts/docker-compose/ngap-tester') {
@@ -292,6 +294,15 @@ pipeline {
         }
       }
     }
+    stage ('Mandatory Tests Check') {
+      steps {
+        script {
+          // Here it will test if all the mandatory tests passed
+          // If not, it will return an error and make the whole pipeline fail
+          sh './ci-scripts/checkNgapTesterHtmlReport.py --job_name ' + JOB_NAME + ' --job_id ' + BUILD_ID + ' --job_url ' + BUILD_URL
+        }
+      }
+    }
   }
   post {
     always {
@@ -301,8 +312,11 @@ pipeline {
         // Currently no public artifacting on the logs and pcap
         sh "chmod 666 cn5g_fed_ngap_test.zip"
         sh "cp cn5g_fed_ngap_test.zip /opt/ngap-tester-logs/cn5g_fed-${JOB_NAME}-${BUILD_ID}.zip"
-        // We still generate HTML report
-        sh './ci-scripts/checkNgapTesterHtmlReport.py --job_name ' + JOB_NAME + ' --job_id ' + BUILD_ID + ' --job_url ' + BUILD_URL
+        // We generate HTML report if we did not already
+        // we shall not fail in post section
+        if (!fileExists('test_results_oai_cn5g_ngap_tester.html')) {
+          sh './ci-scripts/checkNgapTesterHtmlReport.py --job_name ' + JOB_NAME + ' --job_id ' + BUILD_ID + ' --job_url ' + BUILD_URL + ' || true'
+        }
         if (fileExists('test_results_oai_cn5g_ngap_tester.html')) {
           archiveArtifacts artifacts: 'test_results_oai_cn5g_ngap_tester.html'
         }
@@ -378,7 +392,6 @@ def cleanUpDockerCompose() {
 def retrieveLogs(logPath, dcFileName) {
   sh "mkdir -p ${logPath}"
   sh "cp ${dcFileName} ${logPath}"
-  sh "cp /tmp/oai-cn5g-ngap-tester.* ${logPath} || true"
   sh "docker logs oai-nrf > ${logPath}/oai-nrf.log 2>&1"
   sh "docker logs oai-udr > ${logPath}/oai-udr.log 2>&1"
   sh "docker logs oai-udm > ${logPath}/oai-udm.log 2>&1"

ci-scripts/checkNgapTesterHtmlReport.py

@@ -53,10 +53,16 @@ class HtmlReport():
 
     def generate(self, args):
         cwd = os.getcwd()
+        status = True
         with open(os.path.join(cwd, REPORT_NAME), 'w') as wfile:
             wfile.write(generate_header(args))
-            wfile.write(self.testSummary('NGAP-Tester'))
+            (status, testSummary) = self.testSummary('NGAP-Tester')
+            wfile.write(testSummary)
             wfile.write(generate_footer())
+        if status:
+            sys.exit(0)
+        else:
+            sys.exit(-1)
 
     def testSummary(self, testName):
         cwd = os.getcwd()
@@ -103,8 +109,16 @@ class HtmlReport():
             imageDetailsFile.close()
             deployedContainerImages.append((containerName, f'{imageRootName}{imageTag}', imageSize, imageDate))
 
+        mandatoryTests = []
+        if os.path.isfile(cwd + '/ci-scripts/docker-compose/ngap-tester/list-mandatory.txt'):
+            print('found the list of mandatory tests')
+            with open(cwd + '/ci-scripts/docker-compose/ngap-tester/list-mandatory.txt','r') as mandatoryFile:
+                for line in mandatoryFile:
+                    mandatoryTests.append(line.strip())
+
         testCaseDetails = []
         globalStatus = True
+        globalMandatoryStatus = True
         for log_file in log_files:
             if not log_file.endswith(".log"):
                 continue
@@ -115,6 +129,14 @@ class HtmlReport():
             testCaseName = re.sub('.log.*$', '', log_file)
             stringStatus = 'UNKNOWN'
             description  = 'UNKNOWN'
+            if len(mandatoryTests) > 0:
+                try:
+                    index = mandatoryTests.index(testCaseName)
+                    mandatory = True
+                except:
+                    mandatory = False
+            else:
+                mandatory = False
             with open(cwd + '/archives/' + log_file,'r') as imageDetailsFile:
                 for line in imageDetailsFile:
                     result = re.search('Scenario *: Status *: Description', line)
@@ -128,15 +150,22 @@ class HtmlReport():
                         description = result.group('description')
                         description = re.sub('NOT YET VALIDATED - ', '', description)
                         description = re.sub('NOT YET VALIDATED, HAVE TO BE IMPLEMENTED IN OAI CN -', '', description)
-            if not testCaseEnded or not testCaseStatus:
+            if (not testCaseEnded or not testCaseStatus):
                 globalStatus = False
-            testCaseDetails.append((testCaseName, testCaseStatus, stringStatus, description))
+                if mandatory:
+                    globalMandatoryStatus = False
+            testCaseDetails.append((testCaseName, testCaseStatus, mandatory, stringStatus, description))
         testDetails = ''
         if globalStatus:
             message = f'All Tests Passed'
+        elif globalMandatoryStatus and len(mandatoryTests) > 0:
+            message = f'All Mandatory Tests Passed'
         else:
             message = f'Some Tests Failed'
-        testDetails += generate_chapter(f'Load Test Summary for {testName}', message, globalStatus)
+        if len(mandatoryTests) > 0:
+            testDetails += generate_chapter(f'Load Test Summary for {testName}', message, globalMandatoryStatus)
+        else:
+            testDetails += generate_chapter(f'Load Test Summary for {testName}', message, globalStatus)
         testDetails += generate_button_header(f'tc-suite-details', 'More details on ngap-tester results')
         testDetails += generate_image_table_header()
         for (cName,iTag,iSize,iDate) in deployedContainerImages:
@@ -145,7 +174,9 @@ class HtmlReport():
                 testDetails += generate_image_table_separator()
         testDetails += generate_image_table_footer()
         testDetails += generate_list_header()
-        for (name, status, stringStatus, description) in testCaseDetails:
+        for (name, status, mandatory, stringStatus, description) in testCaseDetails:
+            if mandatory and len(mandatoryTests) > 0:
+                name += ' <span class="label label-default">MANDATORY</span>'
             if status:
                 testDetails += generate_list_row(f'{name}', 'info-sign')
             else:
@@ -159,7 +190,12 @@ class HtmlReport():
         testDetails += generate_list_footer()
         testDetails += generate_list_row(f'Logs on private CI server at `oaicicd@selfix:/opt/ngap-tester-logs/cn5g_fed-{args.job_name}-{args.job_id}.zip`', 'info-sign')
         testDetails += generate_button_footer()
-        return testDetails
+
+        # If there is no mandatory list, the test is always passing
+        if len(mandatoryTests) == 0:
+            globalMandatoryStatus = True
+
+        return (globalMandatoryStatus, testDetails)
 
 def _parse_args() -> argparse.Namespace:
     """Parse the command line args

ci-scripts/html-templates/file-header.htm

-<!DOCTYPE html>
-<html class="no-js" lang="en-US">
-<head>
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
-  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
-  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
-  <title>OAI Core Network Test Results for JOB_NAME job build #BUILD_ID</title>
-  <style>
-    .badge-primary {
-   color: #ebeef0;
-   background-color: #000080;
-}
-    .badge-danger {
-   color: #ebeef0;
-   background-color: #FF0000;
-}
-  </style>
-</head>
-<body>
-  <div class="container-fluid" style="margin-left:1em; margin-right:1em">
-  <table width = "100%" style="border-collapse: collapse; border: none;">
-   <tr style="border-collapse: collapse; border: none;">
-   <td style="border-collapse: collapse; border: none;">
-     <a href="http://www.openairinterface.org/">
-        <img src="http://www.openairinterface.org/wp-content/uploads/2016/03/cropped-oai_final_logo2.png" alt="" border="none" height=50 width=150>
-        </img>
-     </a>
-   </td>
-   <td style="border-collapse: collapse; border: none; vertical-align: center;">
-     <b><font size = "6">Job Summary -- Job: JOB_NAME -- Build-ID: <a href="BUILD_URL">BUILD_ID</a></font></b>
-   </td>
-   </tr>
-  </table>
-  <br>