diff --git a/ci-scripts/checkCodingFormattingRules.sh b/ci-scripts/checkCodingFormattingRules.sh index a805f765dfb9b4ca7c6e3cf481a9f0784c08d722..c29941cade3d2464f591ed3a5873341bacb63a4c 100755 --- a/ci-scripts/checkCodingFormattingRules.sh +++ b/ci-scripts/checkCodingFormattingRules.sh @@ -59,7 +59,7 @@ if [ $# -eq 0 ] then echo " ---- Checking the whole repository ----" echo "" - NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive *.c *.h | grep -c Formatted || true` + NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive --exclude=ci-scripts --exclude=cmake_targets *.c *.h | grep -c Formatted || true` echo "Nb Files that do NOT follow OAI rules: $NB_FILES_TO_FORMAT" echo $NB_FILES_TO_FORMAT > ./oai_rules_result.txt @@ -136,7 +136,7 @@ fi # Merge request scenario MERGE_COMMMIT=`git log -n1 --pretty=format:%H` -TARGET_INIT_COMMIT=`cat .git/refs/remotes/origin/$TARGET_BRANCH` +TARGET_INIT_COMMIT=`git log -n1 --pretty=format:%H origin/$TARGET_BRANCH` echo " ---- Checking the modified files by the merge request ----" echo "" diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index d6796ad414e1577e0c9d233e322e46f570f8838f..5b45b48ba67bd0454716428cf89d29cddcf16a97 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -37,6 +37,7 @@ import logging import os from pathlib import Path import time +from multiprocessing import Process, Lock, SimpleQueue #----------------------------------------------------------- # OAI Testing modules @@ -237,3 +238,205 @@ class StaticCodeAnalysis(): return 0 + def LicenceAndFormattingCheck(self, HTML): + if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Building on server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + self.testCase_id = HTML.testCase_id + + # on RedHat/CentOS .git extension is mandatory + result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + if result is not None: + full_ran_repo_name = self.ranRepository.replace('git/', 'git') + else: + full_ran_repo_name = self.ranRepository + '.git' + mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) + mySSH.command('cd ' + lSourcePath, '\$', 5) + mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + # Raphael: here add a check if git clone or git fetch went smoothly + mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + + mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + mySSH.command('mkdir -p cmake_targets/log', '\$', 5) + # if the commit ID is provided use it to point to it + if self.ranCommitID != '': + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should already been checked earlier + argToPass = '' + if (self.ranAllowMerge): + argToPass = '--build-arg MERGE_REQUEST=true --build-arg SRC_BRANCH=' + self.ranBranch + if self.ranTargetBranch == '': + if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + argToPass += ' --build-arg TARGET_BRANCH=develop ' + else: + logging.debug('Merging with the target branch: ' + self.ranTargetBranch) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + argToPass += ' --build-arg TARGET_BRANCH=' + self.ranTargetBranch + ' ' + + mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60) + mySSH.command('docker build --target oai-formatting-check --tag oai-formatting-check:latest ' + argToPass + '--file ci-scripts/docker/Dockerfile.formatting.bionic . > cmake_targets/log/oai-formatting-check.txt 2>&1', '\$', 600) + + mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60) + mySSH.command('docker image prune --force', '\$', 60) + mySSH.command('docker volume prune --force', '\$', 60) + + # Analyzing the logs + mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) + mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) + mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) + mySSH.close() + + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '/*', '.') + + finalStatus = 0 + if (os.path.isfile('./oai-formatting-check.txt')): + analyzed = False + nbFilesNotFormatted = 0 + listFiles = False + listFilesNotFormatted = [] + circularHeaderDependency = False + circularHeaderDependencyFiles = [] + gnuGplLicence = False + gnuGplLicenceFiles = [] + suspectLicence = False + suspectLicenceFiles = [] + with open('./oai-formatting-check.txt', 'r') as logfile: + for line in logfile: + ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line)) + if ret is not None: + analyzed = True + if analyzed: + ret = re.search('Nb Files that do NOT follow OAI rules: (?P<nb_errors>[0-9\.]+)', str(line)) + if ret is not None: + nbFilesNotFormatted = int(ret.group('nb_errors')) + + if re.search('=== Files not properly formatted ===', str(line)) is not None: + listFiles = True + if listFiles: + if re.search('Removing intermediate container', str(line)) is not None: + listFiles = False + elif re.search('Running in|Files not properly formatted', str(line)) is not None: + pass + else: + listFilesNotFormatted.append(str(line).strip()) + + if re.search('=== Files with incorrect define protection ===', str(line)) is not None: + circularHeaderDependency = True + if circularHeaderDependency: + if re.search('Removing intermediate container', str(line)) is not None: + circularHeaderDependency = False + elif re.search('Running in|Files with incorrect define protection', str(line)) is not None: + pass + else: + circularHeaderDependencyFiles.append(str(line).strip()) + + if re.search('=== Files with a GNU GPL licence Banner ===', str(line)) is not None: + gnuGplLicence = True + if gnuGplLicence: + if re.search('Removing intermediate container', str(line)) is not None: + gnuGplLicence = False + elif re.search('Running in|Files with a GNU GPL licence Banner', str(line)) is not None: + pass + else: + gnuGplLicenceFiles.append(str(line).strip()) + + if re.search('=== Files with a suspect Banner ===', str(line)) is not None: + suspectLicence = True + if suspectLicence: + if re.search('Removing intermediate container', str(line)) is not None: + suspectLicence = False + elif re.search('Running in|Files with a suspect Banner', str(line)) is not None: + pass + else: + suspectLicenceFiles.append(str(line).strip()) + + logfile.close() + if analyzed: + logging.debug('files not formatted properly: ' + str(nbFilesNotFormatted)) + if nbFilesNotFormatted == 0: + HTML.CreateHtmlTestRow('File(s) Format', 'OK', CONST.ALL_PROCESSES_OK) + else: + html_queue = SimpleQueue() + html_cell = '<pre style="background-color:white">\n' + html_cell += 'Number of files not following OAI Rules: ' + str(nbFilesNotFormatted) + '\n' + for nFile in listFilesNotFormatted: + html_cell += str(nFile).strip() + '\n' + html_cell += '</pre>' + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue('File(s) Format', 'KO', 1, html_queue) + del(html_cell) + del(html_queue) + + logging.debug('header files not respecting the circular dependency protection: ' + str(len(circularHeaderDependencyFiles))) + if len(circularHeaderDependencyFiles) == 0: + HTML.CreateHtmlTestRow('Header Circular Dependency', 'OK', CONST.ALL_PROCESSES_OK) + else: + html_queue = SimpleQueue() + html_cell = '<pre style="background-color:white">\n' + html_cell += 'Number of files not respecting: ' + str(len(circularHeaderDependencyFiles)) + '\n' + for nFile in circularHeaderDependencyFiles: + html_cell += str(nFile).strip() + '\n' + html_cell += '</pre>' + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue('Header Circular Dependency', 'KO', 1, html_queue) + del(html_cell) + del(html_queue) + finalStatus = -1 + + logging.debug('files with a GNU GPL license: ' + str(len(gnuGplLicenceFiles))) + if len(gnuGplLicenceFiles) == 0: + HTML.CreateHtmlTestRow('Files w/ GNU GPL License', 'OK', CONST.ALL_PROCESSES_OK) + else: + html_queue = SimpleQueue() + html_cell = '<pre style="background-color:white">\n' + html_cell += 'Number of files not respecting: ' + str(len(gnuGplLicenceFiles)) + '\n' + for nFile in gnuGplLicenceFiles: + html_cell += str(nFile).strip() + '\n' + html_cell += '</pre>' + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue('Files w/ GNU GPL License', 'KO', 1, html_queue) + del(html_cell) + del(html_queue) + finalStatus = -1 + + logging.debug('files with a suspect license: ' + str(len(suspectLicenceFiles))) + if len(suspectLicenceFiles) == 0: + HTML.CreateHtmlTestRow('Files with suspect license', 'OK', CONST.ALL_PROCESSES_OK) + else: + html_queue = SimpleQueue() + html_cell = '<pre style="background-color:white">\n' + html_cell += 'Number of files not respecting: ' + str(len(suspectLicenceFiles)) + '\n' + for nFile in suspectLicenceFiles: + html_cell += str(nFile).strip() + '\n' + html_cell += '</pre>' + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue('Files with suspect license', 'KO', 1, html_queue) + del(html_cell) + del(html_queue) + finalStatus = -1 + + else: + finalStatus = -1 + HTML.htmleNBFailureMsg = 'Could not fully analyze oai-formatting-check.txt file' + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) + else: + finalStatus = -1 + HTML.htmleNBFailureMsg = 'Could not access oai-formatting-check.txt file' + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) + + return finalStatus diff --git a/ci-scripts/docker/Dockerfile.formatting.bionic b/ci-scripts/docker/Dockerfile.formatting.bionic new file mode 100644 index 0000000000000000000000000000000000000000..01417a3bdb3c68385cac0141a971d7ab90002c21 --- /dev/null +++ b/ci-scripts/docker/Dockerfile.formatting.bionic @@ -0,0 +1,31 @@ +FROM ubuntu:bionic AS oai-formatting-check + +ARG MERGE_REQUEST +ARG SRC_BRANCH +ARG TARGET_BRANCH +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade --yes && \ + DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + astyle \ + gawk \ + git + +WORKDIR /oai-ran +COPY . . + +RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then echo 'Source Branch = $SRC_BRANCH'; echo 'Target Branch = $TARGET_BRANCH'; else echo 'Push to develop'; fi" +RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then ./ci-scripts/checkCodingFormattingRules.sh --src-branch $SRC_BRANCH --target-branch $TARGET_BRANCH; else ./ci-scripts/checkCodingFormattingRules.sh; fi" + +RUN echo "=== Files not properly formatted ===" && \ + /bin/bash -c "if [[ -f oai_rules_result_list.txt ]]; then cat oai_rules_result_list.txt; fi" + +RUN echo "=== Files with incorrect define protection ===" && \ + /bin/bash -c "if [[ -f header-files-w-incorrect-define.txt ]]; then cat header-files-w-incorrect-define.txt; fi" + +RUN echo "=== Files with a GNU GPL licence Banner ===" && \ + /bin/bash -c "if [[ -f files-w-gnu-gpl-license-banner.txt ]]; then cat files-w-gnu-gpl-license-banner.txt; fi" + +RUN echo "=== Files with a suspect Banner ===" && \ + /bin/bash -c "if [[ -f files-w-suspect-banner.txt ]]; then cat files-w-suspect-banner.txt; fi" diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 0fc37d4ead75759dfacecc3474be9e956afe1dc4..0e0afefaaf7114d421bde4b02f0a2ff31727e1d3 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -255,16 +255,19 @@ class EPCManagement(): mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed-v1.3/docker-compose', '\$', 5) mySSH.command('python3 ./core-network.py '+self.cfgDeploy, '\$', 60) - time.sleep(2) - mySSH.command('docker-compose -p 5gcn ps -a', '\$', 60) + if re.search('start-mini-as-ue', self.cfgDeploy): + dFile = 'docker-compose-mini-nrf-asue.yaml' + else: + dFile = 'docker-compose-mini-nrf.yaml' + mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' ps -a', '\$', 60) if mySSH.getBefore().count('Up (healthy)') != 6: logging.error('Not all container healthy') else: - logging.debug('OK') - mySSH.command('docker-compose config | grep --colour=never image', '\$', 10) + logging.debug('OK --> all containers are healthy') + mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' config | grep --colour=never image', '\$', 10) listOfImages = mySSH.getBefore() for imageLine in listOfImages.split('\\r\\n'): - res1 = re.search('image: (?P<name>[a-zA-Z0-9\-]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine)) + res1 = re.search('image: (?P<name>[a-zA-Z0-9\-/]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine)) res2 = re.search('mysql', str(imageLine)) if res1 is not None and res2 is None: html_cell += res1.group('name') + ':' + res1.group('tag') + ' ' @@ -536,7 +539,7 @@ class EPCManagement(): mySSH.command('python3 ./core-network.py '+self.cfgUnDeploy, '\$', 60) mySSH.command('docker volume prune --force || true', '\$', 60) time.sleep(2) - mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30) + mySSH.command('tshark -r /tmp/oai-cn5g-v1.3.pcap | egrep --colour=never "Tracking area update" ','\$', 30) result = re.search('Tracking area update request', mySSH.getBefore()) if result is not None: message = 'UE requested ' + str(mySSH.getBefore().count('Tracking area update request')) + 'Tracking area update request(s)' @@ -830,8 +833,8 @@ class EPCManagement(): mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60) elif re.match('OAICN5G', self.Type, re.IGNORECASE): mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) - mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30) - mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30) + mySSH.command('cp -f /tmp/oai-cn5g-v1.3.pcap .','\$', 30) + mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g*.pcap','\$', 30) mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip mme.log.zip mme*.log', '\$', 60) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 03947bdfc968c83f83eed43a6f0ab05212b6c218..adef8859423514c45d7b110b4ea2430376b43e5d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -799,7 +799,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.startTime=int(round(time.time() * 1000)) while CiTestObj.FailReportCnt < CiTestObj.repeatCounts[0] and RAN.prematureExit: RAN.prematureExit=False - # At every iteratin of the retry loop, a separator will be added + # At every iteration of the retry loop, a separator will be added # pass CiTestObj.FailReportCnt as parameter of HTML.CreateHtmlRetrySeparator HTML.CreateHtmlRetrySeparator(CiTestObj.FailReportCnt) for test_case_id in todo_tests: @@ -923,6 +923,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CONTAINERS.UndeployObject(HTML, RAN) elif action == 'Cppcheck_Analysis': SCA.CppCheckAnalysis(HTML) + elif action == 'LicenceAndFormattingCheck': + ret = SCA.LicenceAndFormattingCheck(HTML) + if ret != 0: + RAN.prematureExit = True elif action == 'Deploy_Run_PhySim': PHYSIM.Deploy_PhySim(HTML, RAN) elif action == 'DeployGenObject': diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index d6f75191a8da8bc7818d9c7d55adc8b9aad5d21d..b305aff42437b6797f9677155ad4b15adfbb1f9b 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -47,3 +47,4 @@ - PingFromContainer - IperfFromContainer - StatsFromGenObject + - LicenceAndFormattingCheck diff --git a/ci-scripts/xml_files/formatting_check.xml b/ci-scripts/xml_files/formatting_check.xml new file mode 100644 index 0000000000000000000000000000000000000000..bdf47af65d2b67fd92f003f8e1108f9b36eb63f3 --- /dev/null +++ b/ci-scripts/xml_files/formatting_check.xml @@ -0,0 +1,37 @@ +<!-- + + Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The OpenAirInterface Software Alliance licenses this file to You under + the OAI Public License, Version 1.1 (the "License"); you may not use this file + except in compliance with the License. + You may obtain a copy of the License at + + http://www.openairinterface.org/?page_id=698 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + For more information about the OpenAirInterface (OAI) Software Alliance: + contact@openairinterface.org + +--> +<testCaseList> + <htmlTabRef>formatting-tab</htmlTabRef> + <htmlTabName>License and Formatting Checks</htmlTabName> + <htmlTabIcon>wrench</htmlTabIcon> + <TestCaseRequestedList> + 000002 + </TestCaseRequestedList> + <TestCaseExclusionList></TestCaseExclusionList> + + <testCase id="000002"> + <class>LicenceAndFormattingCheck</class> + <desc>License and Formatting Checks</desc> + </testCase> + +</testCaseList>