Merge remote-tracking branch 'origin/ci-fix-cn5g-deploy' into integration_2022_wk19

ci-scripts/checkCodingFormattingRules.sh

@@ -59,7 +59,7 @@ if [ $# -eq 0 ]
 then
     echo " ---- Checking the whole repository ----"
     echo ""
-    NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive *.c *.h | grep -c Formatted || true`
+    NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive --exclude=ci-scripts --exclude=cmake_targets *.c *.h | grep -c Formatted || true`
     echo "Nb Files that do NOT follow OAI rules: $NB_FILES_TO_FORMAT"
     echo $NB_FILES_TO_FORMAT > ./oai_rules_result.txt
 
@@ -136,7 +136,7 @@ fi
 # Merge request scenario
 
 MERGE_COMMMIT=`git log -n1 --pretty=format:%H`
-TARGET_INIT_COMMIT=`cat .git/refs/remotes/origin/$TARGET_BRANCH`
+TARGET_INIT_COMMIT=`git log -n1 --pretty=format:%H origin/$TARGET_BRANCH`
 
 echo " ---- Checking the modified files by the merge request ----"
 echo ""

ci-scripts/cls_static_code_analysis.py

@@ -37,6 +37,7 @@ import logging
 import os
 from pathlib import Path
 import time
+from multiprocessing import Process, Lock, SimpleQueue
 
 #-----------------------------------------------------------
 # OAI Testing modules
@@ -237,3 +238,205 @@ class StaticCodeAnalysis():
 
 		return 0
 
+	def LicenceAndFormattingCheck(self, HTML):
+		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		lIpAddr = self.eNBIPAddress
+		lUserName = self.eNBUserName
+		lPassWord = self.eNBPassword
+		lSourcePath = self.eNBSourceCodePath
+
+		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
+			HELP.GenericHelp(CONST.Version)
+			sys.exit('Insufficient Parameter')
+		logging.debug('Building on server: ' + lIpAddr)
+		mySSH = SSH.SSHConnection()
+		mySSH.open(lIpAddr, lUserName, lPassWord)
+
+		self.testCase_id = HTML.testCase_id
+
+		# on RedHat/CentOS .git extension is mandatory
+		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
+		if result is not None:
+			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
+		else:
+			full_ran_repo_name = self.ranRepository + '.git'
+		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
+		mySSH.command('cd ' + lSourcePath, '\$', 5)
+		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
+		# Raphael: here add a check if git clone or git fetch went smoothly
+		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
+		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
+
+		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
+		mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
+		# if the commit ID is provided use it to point to it
+		if self.ranCommitID != '':
+			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
+		# if the branch is not develop, then it is a merge request and we need to do
+		# the potential merge. Note that merge conflicts should already been checked earlier
+		argToPass = ''
+		if (self.ranAllowMerge):
+			argToPass = '--build-arg MERGE_REQUEST=true --build-arg SRC_BRANCH=' + self.ranBranch
+			if self.ranTargetBranch == '':
+				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
+					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
+					argToPass += ' --build-arg TARGET_BRANCH=develop '
+			else:
+				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
+				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
+				argToPass += ' --build-arg TARGET_BRANCH=' + self.ranTargetBranch + ' '
+
+		mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
+		mySSH.command('docker build --target oai-formatting-check --tag oai-formatting-check:latest ' + argToPass + '--file ci-scripts/docker/Dockerfile.formatting.bionic . > cmake_targets/log/oai-formatting-check.txt 2>&1', '\$', 600)
+
+		mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
+		mySSH.command('docker image prune --force', '\$', 60)
+		mySSH.command('docker volume prune --force', '\$', 60)
+
+		# Analyzing the logs
+		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
+		mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
+		mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
+		mySSH.close()
+
+		mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '/*', '.')
+
+		finalStatus = 0
+		if (os.path.isfile('./oai-formatting-check.txt')):
+			analyzed = False
+			nbFilesNotFormatted = 0
+			listFiles = False
+			listFilesNotFormatted = []
+			circularHeaderDependency = False
+			circularHeaderDependencyFiles = []
+			gnuGplLicence = False
+			gnuGplLicenceFiles = []
+			suspectLicence = False
+			suspectLicenceFiles = []
+			with open('./oai-formatting-check.txt', 'r') as logfile:
+				for line in logfile:
+					ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
+					if ret is not None:
+						analyzed = True
+					if analyzed:
+						ret = re.search('Nb Files that do NOT follow OAI rules: (?P<nb_errors>[0-9\.]+)', str(line))
+						if ret is not None:
+							nbFilesNotFormatted = int(ret.group('nb_errors'))
+
+						if re.search('=== Files not properly formatted ===', str(line)) is not None:
+							listFiles = True
+						if listFiles:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								listFiles = False
+							elif re.search('Running in|Files not properly formatted', str(line)) is not None:
+								pass
+							else:
+								listFilesNotFormatted.append(str(line).strip())
+
+						if re.search('=== Files with incorrect define protection ===', str(line)) is not None:
+							circularHeaderDependency = True
+						if circularHeaderDependency:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								circularHeaderDependency = False
+							elif re.search('Running in|Files with incorrect define protection', str(line)) is not None:
+								pass
+							else:
+								circularHeaderDependencyFiles.append(str(line).strip())
+
+						if re.search('=== Files with a GNU GPL licence Banner ===', str(line)) is not None:
+							gnuGplLicence = True
+						if gnuGplLicence:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								gnuGplLicence = False
+							elif re.search('Running in|Files with a GNU GPL licence Banner', str(line)) is not None:
+								pass
+							else:
+								gnuGplLicenceFiles.append(str(line).strip())
+
+						if re.search('=== Files with a suspect Banner ===', str(line)) is not None:
+							suspectLicence = True
+						if suspectLicence:
+							if re.search('Removing intermediate container', str(line)) is not None:
+								suspectLicence = False
+							elif re.search('Running in|Files with a suspect Banner', str(line)) is not None:
+								pass
+							else:
+								suspectLicenceFiles.append(str(line).strip())
+
+				logfile.close()
+			if analyzed:
+				logging.debug('files not formatted properly: ' + str(nbFilesNotFormatted))
+				if nbFilesNotFormatted == 0:
+					HTML.CreateHtmlTestRow('File(s) Format', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not following OAI Rules: ' + str(nbFilesNotFormatted) + '\n'
+					for nFile in listFilesNotFormatted:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('File(s) Format', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+
+				logging.debug('header files not respecting the circular dependency protection: ' + str(len(circularHeaderDependencyFiles)))
+				if len(circularHeaderDependencyFiles) == 0:
+					HTML.CreateHtmlTestRow('Header Circular Dependency', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(circularHeaderDependencyFiles)) + '\n'
+					for nFile in circularHeaderDependencyFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Header Circular Dependency', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+				logging.debug('files with a GNU GPL license: ' + str(len(gnuGplLicenceFiles)))
+				if len(gnuGplLicenceFiles) == 0:
+					HTML.CreateHtmlTestRow('Files w/ GNU GPL License', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(gnuGplLicenceFiles)) + '\n'
+					for nFile in gnuGplLicenceFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Files w/ GNU GPL License', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+				logging.debug('files with a suspect license: ' + str(len(suspectLicenceFiles)))
+				if len(suspectLicenceFiles) == 0:
+					HTML.CreateHtmlTestRow('Files with suspect license', 'OK', CONST.ALL_PROCESSES_OK)
+				else:
+					html_queue = SimpleQueue()
+					html_cell = '<pre style="background-color:white">\n'
+					html_cell += 'Number of files not respecting: ' + str(len(suspectLicenceFiles)) + '\n'
+					for nFile in suspectLicenceFiles:
+						html_cell += str(nFile).strip() + '\n'
+					html_cell += '</pre>'
+					html_queue.put(html_cell)
+					HTML.CreateHtmlTestRowQueue('Files with suspect license', 'KO', 1, html_queue)
+					del(html_cell)
+					del(html_queue)
+					finalStatus = -1
+
+			else:
+				finalStatus = -1
+				HTML.htmleNBFailureMsg = 'Could not fully analyze oai-formatting-check.txt file'
+				HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+		else:
+			finalStatus = -1
+			HTML.htmleNBFailureMsg = 'Could not access oai-formatting-check.txt file'
+			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+
+		return finalStatus

ci-scripts/docker/Dockerfile.formatting.bionic

+FROM ubuntu:bionic AS oai-formatting-check
+
+ARG MERGE_REQUEST
+ARG SRC_BRANCH
+ARG TARGET_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get upgrade --yes && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+       astyle \
+       gawk \
+       git
+
+WORKDIR /oai-ran
+COPY . .
+
+RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then echo 'Source Branch = $SRC_BRANCH'; echo 'Target Branch = $TARGET_BRANCH'; else echo 'Push to develop'; fi"
+RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then ./ci-scripts/checkCodingFormattingRules.sh --src-branch $SRC_BRANCH --target-branch $TARGET_BRANCH; else ./ci-scripts/checkCodingFormattingRules.sh; fi"
+
+RUN echo "=== Files not properly formatted ===" && \
+    /bin/bash -c "if [[ -f oai_rules_result_list.txt ]]; then cat oai_rules_result_list.txt; fi"
+
+RUN echo "=== Files with incorrect define protection ===" && \
+   /bin/bash -c "if [[ -f header-files-w-incorrect-define.txt ]]; then cat header-files-w-incorrect-define.txt; fi"
+
+RUN echo "=== Files with a GNU GPL licence Banner ===" && \
+   /bin/bash -c "if [[ -f files-w-gnu-gpl-license-banner.txt ]]; then cat files-w-gnu-gpl-license-banner.txt; fi"
+
+RUN echo "=== Files with a suspect Banner ===" && \
+  /bin/bash -c "if [[ -f files-w-suspect-banner.txt ]]; then cat files-w-suspect-banner.txt; fi"

ci-scripts/epc.py

@@ -255,16 +255,19 @@ class EPCManagement():
 			mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5)
 			mySSH.command('cd /opt/oai-cn5g-fed-v1.3/docker-compose', '\$', 5)
 			mySSH.command('python3 ./core-network.py '+self.cfgDeploy, '\$', 60)
-			time.sleep(2)
-			mySSH.command('docker-compose -p 5gcn ps -a', '\$', 60)
+			if re.search('start-mini-as-ue', self.cfgDeploy):
+				dFile = 'docker-compose-mini-nrf-asue.yaml'
+			else:
+				dFile = 'docker-compose-mini-nrf.yaml'
+			mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' ps -a', '\$', 60)
 			if mySSH.getBefore().count('Up (healthy)') != 6:
 				logging.error('Not all container healthy')
 			else:
-				logging.debug('OK')
-			mySSH.command('docker-compose config | grep --colour=never image', '\$', 10)
+				logging.debug('OK --> all containers are healthy')
+			mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' config | grep --colour=never image', '\$', 10)
 			listOfImages = mySSH.getBefore()
 			for imageLine in listOfImages.split('\\r\\n'):
-				res1 = re.search('image: (?P<name>[a-zA-Z0-9\-]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
+				res1 = re.search('image: (?P<name>[a-zA-Z0-9\-/]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
 				res2 = re.search('mysql', str(imageLine))
 				if res1 is not None and res2 is None:
 					html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
@@ -536,7 +539,7 @@ class EPCManagement():
 			mySSH.command('python3 ./core-network.py '+self.cfgUnDeploy, '\$', 60)
 			mySSH.command('docker volume prune --force || true', '\$', 60)
 			time.sleep(2)
-			mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
+			mySSH.command('tshark -r /tmp/oai-cn5g-v1.3.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
 			result = re.search('Tracking area update request', mySSH.getBefore())
 			if result is not None:
 				message = 'UE requested ' + str(mySSH.getBefore().count('Tracking area update request')) + 'Tracking area update request(s)'
@@ -830,8 +833,8 @@ class EPCManagement():
 				mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60)
 		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
 			mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5)
-			mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30)
-			mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30)
+			mySSH.command('cp -f /tmp/oai-cn5g-v1.3.pcap .','\$', 30)
+			mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g*.pcap','\$', 30)
 			mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30)
 		elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
 			mySSH.command('zip mme.log.zip mme*.log', '\$', 60)

ci-scripts/main.py

@@ -799,7 +799,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	HTML.startTime=int(round(time.time() * 1000))
 	while CiTestObj.FailReportCnt < CiTestObj.repeatCounts[0] and RAN.prematureExit:
 		RAN.prematureExit=False
-		# At every iteratin of the retry loop, a separator will be added
+		# At every iteration of the retry loop, a separator will be added
 		# pass CiTestObj.FailReportCnt as parameter of HTML.CreateHtmlRetrySeparator
 		HTML.CreateHtmlRetrySeparator(CiTestObj.FailReportCnt)
 		for test_case_id in todo_tests:
@@ -923,6 +923,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					CONTAINERS.UndeployObject(HTML, RAN)
 				elif action == 'Cppcheck_Analysis':
 					SCA.CppCheckAnalysis(HTML)
+				elif action == 'LicenceAndFormattingCheck':
+					ret = SCA.LicenceAndFormattingCheck(HTML)
+					if ret != 0:
+						RAN.prematureExit = True
 				elif action == 'Deploy_Run_PhySim':
 					PHYSIM.Deploy_PhySim(HTML, RAN)
 				elif action == 'DeployGenObject':

ci-scripts/xml_class_list.yml

@@ -47,3 +47,4 @@
   - PingFromContainer
   - IperfFromContainer
   - StatsFromGenObject
+  - LicenceAndFormattingCheck

ci-scripts/xml_files/formatting_check.xml

+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+	<htmlTabRef>formatting-tab</htmlTabRef>
+	<htmlTabName>License and Formatting Checks</htmlTabName>
+	<htmlTabIcon>wrench</htmlTabIcon>
+	<TestCaseRequestedList>
+ 000002
+	</TestCaseRequestedList>
+	<TestCaseExclusionList></TestCaseExclusionList>
+
+	<testCase id="000002">
+		<class>LicenceAndFormattingCheck</class>
+		<desc>License and Formatting Checks</desc>
+	</testCase>
+
+</testCaseList>