chore(ci): starting collecting logs from pods from deployment

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/Jenkinsfile-GitLab-COTS-UE-Test

@@ -163,6 +163,7 @@ pipeline {
           sleep 60
           sh "oc describe pod &>> archives/describe-pods.logs"
           sh "oc get pods -o wide"
+          CollectLogsFromPods()
           coreStatus = checkCoreNetworkDeployment()
           if (coreStatus > 0) {
             error ('Core Deployment went wrong')
@@ -511,7 +512,7 @@ def checkCoreNetworkDeployment() {
   return status
 }
 
-def retrieveLogsFromPods() {
+def CollectLogsFromPods() {
   sh "mkdir -p archives"
   for (ii = 0; ii < imageNames.size(); ii++) {
     podName = sh returnStdout: true, script: "oc get pods | grep ${imageNames[ii]} | awk {'print \$1'} || true"
@@ -520,11 +521,14 @@ def retrieveLogsFromPods() {
       nrfPod = podName
     }
     try {
-      sh "oc logs ${podName} ${containerInPods[ii]} &> archives/${imageNames[ii]}.logs"
+      sh "nohup oc logs -f ${podName} ${containerInPods[ii]} &> archives/${imageNames[ii]}.logs &"
     } catch (Exception e) {
       echo "Getting logs from ${podName} failed"
     }
   }
+}
+
+def retrieveLogsFromPods() {
   // Retrieving mysql logs
   podName = sh returnStdout: true, script: "oc get pods | grep mysql | awk {'print \$1'} || true"
   podName = podName.trim()

ci-scripts/charts/oai-amf/values.yaml

@@ -97,7 +97,7 @@ readinessProbe: false
 
 livenessProbe: false
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 nodeSelector: {}
 

ci-scripts/charts/oai-ausf/values.yaml

@@ -81,7 +81,7 @@ readinessProbe: true
 
 livenessProbe: false
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 nodeSelector: {}
 

ci-scripts/charts/oai-nrf/values.yaml

@@ -73,7 +73,7 @@ resources:
       cpu: 100m
       memory: 128Mi
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 readinessProbe: true
 

ci-scripts/charts/oai-smf/values.yaml

@@ -94,7 +94,7 @@ resources:
 readinessProbe: true
 livenessProbe: false
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 nodeSelector: {}
 

ci-scripts/charts/oai-udm/values.yaml

@@ -79,7 +79,7 @@ readinessProbe: true
 
 livenessProbe: false
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 nodeSelector: {}
 

ci-scripts/charts/oai-upf/values.yaml

@@ -112,7 +112,7 @@ readinessProbe: true
 
 livenessProbe: false
 
-terminationGracePeriodSeconds: 5
+terminationGracePeriodSeconds: 60
 
 nodeSelector: {}
 

ci-scripts/checkCOTS-UE-Testing.py

@@ -255,6 +255,32 @@ def detailsCoreDeployment():
     detailsHtml += generate_button_footer()
     return (status, detailsHtml)
 
+def detailsCoreUndeployment():
+    cwd = os.getcwd()
+    status = True
+    detailsHtml = generate_button_header('cn5g-undeploy-details', 'Details on the OAI CN5G Undeployment')
+    detailsHtml += generate_list_header()
+    coreElements = ['oai-nrf', 'oai-amf', 'oai-smf', 'oai-upf', 'oai-ausf', 'oai-udm', 'oai-udr']
+    for element in coreElements:
+        byeMessagePresent = False
+        if not os.path.isfile(f'{cwd}/archives/{element}.logs'):
+            detailsHtml += generate_list_row(f'{element} has NO LOGS', 'remove-sign')
+            status = False
+            continue
+        with open(f'{cwd}/archives/{element}.logs', mode='r', errors='ignore') as nfRuntimeLogFile:
+            for line in nfRuntimeLogFile:
+                result = re.search('system.*info.* Bye. Shutdown Procedure took (?P<duration>[0-9]+) ms', line)
+                if result is not None:
+                    byeMessagePresent = True
+                    duration = int(result.group('duration'))
+                    detailsHtml += generate_list_row(f'{element} has the bye message -- Shutdown took {duration} ms', 'info-sign')
+        if not byeMessagePresent:
+            detailsHtml += generate_list_row(f'{element} has NO Bye message', 'remove-sign')
+            status = False
+    detailsHtml += generate_list_footer()
+    detailsHtml += generate_button_footer()
+    return (status, detailsHtml)
+
 def checkAMFconnection():
     cwd = os.getcwd()
     count = 0
@@ -446,6 +472,7 @@ if __name__ == '__main__':
         ueStop1Status = True
     else:
         ueStop1Status = False
+    (undeployStatus, undeployDetails) = detailsCoreUndeployment()
 
     cwd = os.getcwd()
     with open(os.path.join(cwd, REPORT_NAME), 'w') as wfile:
@@ -464,7 +491,8 @@ if __name__ == '__main__':
         wfile.write(ueStartTest1)
         wfile.write(generate_chapter('Second COTS-UE Deconnection', 'PDU Session release / Deregistration', ueStop1Status))
         wfile.write(ueStopTest1)
-        wfile.write(generate_chapter('Post-Run PCAP Analysis', 'To be done', True))
+        wfile.write(generate_chapter('Shutdown Analysis', 'Status for undeployment', undeployStatus))
+        wfile.write(undeployDetails)
         wfile.write(generate_footer())
     if not coreStatus or not ranStatus or not ueStart0Status or not ueStop0Status or not ueStart1Status or not ueStop1Status:
         sys.exit(1)