diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 69959011acacfc269885fd98a997baf0ee7cfd54..29d5d761ec3b1523e3b44e89f18621a37a37cde4 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -391,44 +391,13 @@ pipeline { stage ("Test MONOLITHIC - FDD - Band 7 - B210") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - gitlabCommitStatus(name: "Test-Mono-FDD-Band7") { - build job: 'eNB-CI-FDD-Band7-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - } - } else { - gitlabCommitStatus(name: "Test-Mono-FDD-Band7") { - build job: 'eNB-CI-FDD-Band7-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - } - } + triggerSlaveJob ('eNB-CI-FDD-Band7-B210', 'Test-Mono-FDD-Band7') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-eNB-CI-FDD-Band7-B210.html')) { - copyArtifacts(projectName: 'eNB-CI-FDD-Band7-B210', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-eNB-CI-FDD-Band7-B210.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-FDD-Band7-B210.html" - archiveArtifacts artifacts: 'test_results-eNB-CI-FDD-Band7-B210.html' - } - } + finalizeSlaveJob('eNB-CI-FDD-Band7-B210') } } failure { @@ -441,44 +410,13 @@ pipeline { stage ("Test MONOLITHIC - TDD - Band 40 - B210") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - gitlabCommitStatus(name: "Test-Mono-TDD-Band40") { - build job: 'eNB-CI-TDD-Band40-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - } - } else { - gitlabCommitStatus(name: "Test-Mono-TDD-Band40") { - build job: 'eNB-CI-TDD-Band40-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - } - } + triggerSlaveJob ('eNB-CI-TDD-Band40-B210', 'Test-Mono-TDD-Band40') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-eNB-CI-TDD-Band40-B210.html')) { - copyArtifacts(projectName: 'eNB-CI-TDD-Band40-B210', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-eNB-CI-TDD-Band40-B210.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-TDD-Band40-B210.html" - archiveArtifacts artifacts: 'test_results-eNB-CI-TDD-Band40-B210.html' - } - } + finalizeSlaveJob('eNB-CI-TDD-Band40-B210') } } failure { @@ -491,44 +429,13 @@ pipeline { stage ("Test IF4p5 - FDD - Band 7 - B210") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - gitlabCommitStatus(name: "Test-IF4p5-FDD-Band7") { - build job: 'eNB-CI-IF4p5-FDD-Band7-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - } - } else { - gitlabCommitStatus(name: "Test-IF4p5-FDD-Band7") { - build job: 'eNB-CI-IF4p5-FDD-Band7-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - } - } + triggerSlaveJob ('eNB-CI-IF4p5-FDD-Band7-B210', 'Test-IF4p5-FDD-Band7') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-eNB-CI-IF4p5-FDD-Band7-B210.html')) { - copyArtifacts(projectName: 'eNB-CI-IF4p5-FDD-Band7-B210', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-eNB-CI-IF4p5-FDD-Band7-B210.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-IF4p5-FDD-Band7-B210.html" - archiveArtifacts artifacts: 'test_results-eNB-CI-IF4p5-FDD-Band7-B210.html' - } - } + finalizeSlaveJob('eNB-CI-IF4p5-FDD-Band7-B210') } } failure { @@ -541,44 +448,13 @@ pipeline { stage ("Test IF4p5 - TDD - Band 40 - B210") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - gitlabCommitStatus(name: "Test-IF4p5-TDD-Band40") { - build job: 'eNB-CI-IF4p5-TDD-Band40-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - } - } else { - gitlabCommitStatus(name: "Test-IF4p5-TDD-Band40") { - build job: 'eNB-CI-IF4p5-TDD-Band40-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - } - } + triggerSlaveJob ('eNB-CI-IF4p5-TDD-Band40-B210', 'Test-IF4p5-TDD-Band40') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-eNB-CI-IF4p5-TDD-Band40-B210.html')) { - copyArtifacts(projectName: 'eNB-CI-IF4p5-TDD-Band40-B210', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-eNB-CI-IF4p5-TDD-Band40-B210.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-IF4p5-TDD-Band40-B210.html" - archiveArtifacts artifacts: 'test_results-eNB-CI-IF4p5-TDD-Band40-B210.html' - } - } + finalizeSlaveJob('eNB-CI-IF4p5-TDD-Band40-B210') } } failure { @@ -591,44 +467,13 @@ pipeline { stage ("Test MONOLITHIC - FDD - Band 13 - B210") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - gitlabCommitStatus(name: "Test-Mono-FDD-Band13") { - build job: 'eNB-CI-MONO-FDD-Band13-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - } - } else { - gitlabCommitStatus(name: "Test-Mono-FDD-Band13") { - build job: 'eNB-CI-MONO-FDD-Band13-B210', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - } - } + triggerSlaveJob ('eNB-CI-MONO-FDD-Band13-B210', 'Test-Mono-FDD-Band13') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-eNB-CI-MONO-FDD-Band13-B210.html')) { - copyArtifacts(projectName: 'eNB-CI-MONO-FDD-Band13-B210', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-eNB-CI-MONO-FDD-Band13-B210.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-eNB-CI-MONO-FDD-Band13-B210.html" - archiveArtifacts artifacts: 'test_results-eNB-CI-MONO-FDD-Band13-B210.html' - } - } + finalizeSlaveJob('eNB-CI-MONO-FDD-Band13-B210') } } failure { @@ -638,47 +483,19 @@ pipeline { } } } - stage ("Test OAI UE Sniffing - FDD - Band 20 - B200") { + stage ("Test OAI UE - FDD - Bands 20/7 - B200") { steps { script { - if ("MERGE".equals(env.gitlabActionType)) { - //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") { - build job: 'UE-CI-FDD-Band20-B200', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), - string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), - booleanParam(name: 'eNB_mergeRequest', value: true), - string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) - ] - //} - } else { - //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") { - build job: 'UE-CI-FDD-Band20-B200', - parameters: [ - string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), - string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), - booleanParam(name: 'eNB_mergeRequest', value: false) - ] - //} - } + triggerSlaveJobNoGitLab ('UE-CI-FDD-Band20-B200') + // calling a second job sequentially + triggerSlaveJob ('eNB-UE-CI-MONO-FDD-Band7-B200', 'Test-eNB-OAI-UE-FDD-Band7') } } post { - // In case of any non-success, we are retrieving the HTML report of the last completed - // slave job. The only drop-back is that we may retrieve the HTML report of a previous build always { script { - if (!fileExists('test_results-UE-CI-FDD-Band20-B200.html')) { - copyArtifacts(projectName: 'UE-CI-FDD-Band20-B200', - filter: 'test_results*.html', - selector: lastCompleted()) - if (fileExists('test_results-UE-CI-FDD-Band20-B200.html')) { - sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-UE-CI-FDD-Band20-B200.html" - archiveArtifacts artifacts: 'test_results-UE-CI-FDD-Band20-B200.html' - } - } + finalizeSlaveJob('UE-CI-FDD-Band20-B200') + finalizeSlaveJob('eNB-UE-CI-MONO-FDD-Band7-B200') } } failure { @@ -766,3 +583,66 @@ OAI CI Team''', } } } + +// ---- Slave Job functions + +def triggerSlaveJob (jobName, gitlabStatusName) { + if ("MERGE".equals(env.gitlabActionType)) { + gitlabCommitStatus(name: gitlabStatusName) { + build job: jobName, + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), + string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), + booleanParam(name: 'eNB_mergeRequest', value: true), + string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) + ] + } + } else { + gitlabCommitStatus(name: gitlabStatusName) { + build job: jobName, + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), + string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), + booleanParam(name: 'eNB_mergeRequest', value: false) + ] + } + } +} + +def triggerSlaveJobNoGitLab (jobName) { + if ("MERGE".equals(env.gitlabActionType)) { + build job: jobName, + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), + string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), + booleanParam(name: 'eNB_mergeRequest', value: true), + string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) + ] + } else { + build job: jobName, + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), + string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), + booleanParam(name: 'eNB_mergeRequest', value: false) + ] + } +} + +def finalizeSlaveJob(jobName) { + // In case of any non-success, we are retrieving the HTML report of the last completed + // slave job. The only drop-back is that we may retrieve the HTML report of a previous build + fileName = "test_results-${jobName}.html" + if (!fileExists(fileName)) { + copyArtifacts(projectName: jobName, + filter: 'test_results*.html', + selector: lastCompleted()) + if (fileExists(fileName)) { + sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' ${fileName}" + archiveArtifacts artifacts: fileName + } + } +} diff --git a/ci-scripts/Jenkinsfile-tmp-full-ran b/ci-scripts/Jenkinsfile-tmp-full-ran new file mode 100644 index 0000000000000000000000000000000000000000..8e617046895ac20b61133ee0ba8cf7d66eebc9f2 --- /dev/null +++ b/ci-scripts/Jenkinsfile-tmp-full-ran @@ -0,0 +1,376 @@ +#!/bin/groovy +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +// Template Jenkins Declarative Pipeline script to run Test w/ RF HW + +// Location of the python executor node shall be in the same subnet as the others servers +def pythonExecutor = params.pythonExecutor + +// Location of the test XML file to be run +def testXMLFile = params.pythonTestXmlFile +def mainPythonAllXmlFiles = "" +def buildStageStatus = true + +// Name of the test stage +def testStageName = params.pipelineTestStageName + +// Name of the phone resource +def ciSmartPhoneResource = params.smartphonesResource + +// Name of the phone resource +def oaiUEResource = params.oaiUEResource + +// Terminate Status +def termENB = 0 +def termOAIUE = 1 +def termStatusArray = new Boolean[termOAIUE + 1] +termStatusArray[termENB] = false +termStatusArray[termOAIUE] = false + +// Global Parameters. Normally they should be populated when the master job +// triggers the slave job with parameters +def eNB_Repository +def eNB_Branch +def eNB_CommitID +def eNB_AllowMergeRequestProcess = false +def eNB_TargetBranch + +pipeline { + agent { + label pythonExecutor + } + options { + disableConcurrentBuilds() + ansiColor('xterm') + lock(extra: [[resource: ciSmartPhoneResource]], resource: ciSmartPhoneResource) + } + + stages { + stage ("Verify Parameters") { + steps { + script { + echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' + def allParametersPresent = true + + // It is already to late to check it + if (params.pythonExecutor != null) { + echo "eNB CI executor node : ${pythonExecutor}" + } + // If not present picking a default XML file + if (params.pythonTestXmlFile == null) { + // picking default + testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml' + echo "Test XML file(default): ${testXMLFile}" + mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " " + } else { + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists("ci-scripts/" + xmlFile)) { + mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " " + echo "Test XML file : ${xmlFile}" + } + } + } + // If not present picking a default Stage Name + if (params.pipelineTestStageName == null) { + // picking default + testStageName = 'Template Test Stage' + } + + if (params.smartphonesResource == null) { + allParametersPresent = false + } + if (params.oaiUEResource == null) { + allParametersPresent = false + } + if (params.eNB_IPAddress == null) { + allParametersPresent = false + } + if (params.eNB_SourceCodePath == null) { + allParametersPresent = false + } + if (params.eNB_Credentials == null) { + allParametersPresent = false + } + if (params.UE_IPAddress == null) { + allParametersPresent = false + } + if (params.UE_SourceCodePath == null) { + allParametersPresent = false + } + if (params.UE_Credentials == null) { + allParametersPresent = false + } + // the following 4 parameters should be pushed by the master trigger + // if not present, take the job GIT variables (used for developing) + if (params.eNB_Repository == null) { + eNB_Repository = env.GIT_URL + } else { + eNB_Repository = params.eNB_Repository + } + echo "eNB_Repository : ${eNB_Repository}" + if (params.eNB_Branch == null) { + eNB_Branch = env.GIT_BRANCH + } else { + eNB_Branch = params.eNB_Branch + } + echo "eNB_Branch : ${eNB_Branch}" + if (params.eNB_CommitID == null) { + eNB_CommitID = env.GIT_COMMIT + } else { + eNB_CommitID = params.eNB_CommitID + } + echo "eNB_CommitID : ${eNB_CommitID}" + if (params.eNB_mergeRequest != null) { + eNB_AllowMergeRequestProcess = params.eNB_mergeRequest + if (eNB_AllowMergeRequestProcess) { + if (params.eNB_TargetBranch != null) { + eNB_TargetBranch = params.eNB_TargetBranch + } else { + eNB_TargetBranch = 'develop' + } + echo "eNB_TargetBranch : ${eNB_TargetBranch}" + } + } + + if (params.EPC_IPAddress == null) { + allParametersPresent = false + } + if (params.EPC_Type == null) { + allParametersPresent = false + } + if (params.EPC_SourceCodePath == null) { + allParametersPresent = false + } + if (params.EPC_Credentials == null) { + allParametersPresent = false + } + + if (params.ADB_IPAddress == null) { + allParametersPresent = false + } + if (params.ADB_Credentials == null) { + allParametersPresent = false + } + + if (allParametersPresent) { + echo "All parameters are present" + if (eNB_AllowMergeRequestProcess) { + sh "git fetch" + sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest" + } else { + sh "git fetch" + sh "git checkout -f ${eNB_CommitID}" + } + } else { + echo "Some parameters are missing" + sh "./ci-scripts/fail.sh" + } + } + } + } + stage ("Build and Test") { + steps { + script { + dir ('ci-scripts') { + echo "\u2705 \u001B[32m${testStageName}\u001B[0m" + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'] + ]) { + sh "python3 main.py --mode=InitiateHtml --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}" + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists(xmlFile)) { + try { + sh "python3 main.py --mode=TesteNB --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" + } catch (Exception e) { + currentBuild.result = 'FAILURE' + buildStageStatus = false + } + } + } + sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password}" + } + } + } + } + } + stage ("Terminate") { + parallel { + stage('Terminate eNB') { + steps { + echo '\u2705 \u001B[32mTerminate eNB\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=TerminateeNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}" + } + } + post { + success { + script { + termStatusArray[termENB] = true + } + } + } + } + stage('Terminate OAI-UE') { + steps { + echo '\u2705 \u001B[32mTerminate OAI-UE\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=TerminateOAIUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password}" + } + } + post { + success { + script { + termStatusArray[termUE] = true + } + } + } + } + } + } + stage('Log Collection') { + parallel { + stage('Log Collection (eNB - Run)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m' + sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("enb.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "enb.log.${env.BUILD_ID}.zip" + } + if(fileExists("ci-scripts/test_results.html")) { + sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" + sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" + archiveArtifacts "test_results-${JOB_NAME}.html" + } + } + } + } + stage('Log Collection (OAI UE - Run)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (OAI UE - Run)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectOAIUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (OAI UE - Run)\u001B[0m' + sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/ue.log.zip ./ue.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("ue.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "ue.log.${env.BUILD_ID}.zip" + } + } + } + } + stage('Log Collection (Ping)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (Ping)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectPing --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}" + + echo '\u2705 \u001B[32mLog Transfer (Ping)\u001B[0m' + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/ping.log.zip ./ping.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("ping.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "ping.log.${env.BUILD_ID}.zip" + } + } + } + } + stage('Log Collection (Iperf)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (Iperf)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectIperf --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}" + + echo '\u2705 \u001B[32mLog Transfer (Iperf)\u001B[0m' + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/iperf.log.zip ./iperf.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("iperf.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "iperf.log.${env.BUILD_ID}.zip" + } + } + } + } + } + } + } + + post { + always { + script { + if (params.pipelineZipsConsoleLog != null) { + if (params.pipelineZipsConsoleLog) { + echo "Archiving Jenkins console log" + sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true" + sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true" + if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip" + } + } + } + } + } + // Making sure that we really shutdown every thing before leaving + failure { + script { + if (!termStatusArray[termENB]) { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=TerminateeNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}" + } + } + if (!termStatusArray[termOAIUE]) { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=TerminateOAIUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password}" + } + } + } + } + } +} diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 96d7397e46fb1df7f88a66fdf721eb5fdf6bb516..c87e44e375e006008f266375f3d0a43e6c5f242e 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -42,9 +42,6 @@ ENB_PROCESS_SEG_FAULT = -11 ENB_PROCESS_ASSERTION = -12 ENB_PROCESS_REALTIME_ISSUE = -13 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14 -UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20 -UE_PROCESS_COULD_NOT_SYNC = -21 -UE_PROCESS_ASSERTION = -22 HSS_PROCESS_FAILED = -2 HSS_PROCESS_OK = +2 MME_PROCESS_FAILED = -3 @@ -52,6 +49,11 @@ MME_PROCESS_OK = +3 SPGW_PROCESS_FAILED = -4 SPGW_PROCESS_OK = +4 UE_IP_ADDRESS_ISSUE = -5 +OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20 +OAI_UE_PROCESS_COULD_NOT_SYNC = -21 +OAI_UE_PROCESS_ASSERTION = -22 +OAI_UE_PROCESS_FAILED = -6 +OAI_UE_PROCESS_OK = +6 #----------------------------------------------------------- # Import @@ -118,6 +120,7 @@ class SSHConnection(): self.htmlFooterCreated = False self.htmlUEConnected = -1 self.htmleNBFailureMsg = '' + self.htmlUEFailureMsg = '' self.picocom_closure = False self.idle_sleep_time = 0 self.htmlTabRefs = [] @@ -330,7 +333,7 @@ class SSHConnection(): self.command('mkdir -p log', '\$', 5) self.command('chmod 777 log', '\$', 5) # no need to remove in log (git clean did the trick) - self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_enb.log', 'Bypassing the Tests', 600) + self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests', 600) self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) self.command('mv compile_oai_enb.log ' + 'build_log_' + self.testCase_id, '\$', 5) @@ -366,7 +369,7 @@ class SSHConnection(): self.command('mkdir -p log', '\$', 5) self.command('chmod 777 log', '\$', 5) # no need to remove in log (git clean did the trick) - self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600) + self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee compile_oai_ue.log', 'Bypassing the Tests', 600) self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5) @@ -437,8 +440,9 @@ class SSHConnection(): if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = True - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = False + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus) self.CreateHtmlTabFooter(False) @@ -483,13 +487,13 @@ class SSHConnection(): # do not reset board twice in IF4.5 case result = re.search('rru|enb', str(config_file)) if result is not None: - self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 10) result = re.search('type: b200', str(self.ssh.before)) if result is not None: logging.debug('Found a B2xx device --> resetting it') - self.command('echo ' + self.eNBPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware - self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 15) # Make a copy and adapt to EPC / eNB IP addresses self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2); @@ -584,50 +588,36 @@ class SSHConnection(): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() sys.exit('Insufficient Parameter') - #initialize_OAI_UE_flag = True - #pStatus = self.CheckOAIUEProcessExist(initialize_OAI_UE_flag) - #if (pStatus < 0): - # self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus) - # self.CreateHtmlTabFooter(False) - # sys.exit(1) + result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args)) + if result is None: + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) + if (pStatus < 0): + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus) + self.CreateHtmlTabFooter(False) + sys.exit(1) self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) # b2xx_fx3_utils reset procedure - self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 10) result = re.search('type: b200', str(self.ssh.before)) if result is not None: logging.debug('Found a B2xx device --> resetting it') - self.command('echo ' + self.UEPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware - self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 15) else: logging.debug('Did not find any B2xx device') self.command('cd ' + self.UESourceCodePath, '\$', 5) - # Initialize_OAI_UE_args usually start with -C and followed by the location in repository - #full_config_file = self.Initialize_OAI_UE_args.replace('-O ','') - #extIdx = full_config_file.find('.conf') - #if (extIdx > 0): - # extra_options = full_config_file[extIdx + 5:] - # # if tracer options is on, compiling and running T Tracer - # result = re.search('T_stdout', str(extra_options)) - ## if result is not None: - # logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') - # self.command('cd common/utils/T/tracer', '\$', 5) - # self.command('make', '\$', 10) - # self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.UEUserName, 5) - # self.command('cd ' + self.UESourceCodePath, '\$', 5) - # full_config_file = full_config_file[:extIdx + 5] - # config_path, config_file = os.path.split(full_config_file) - #ci_full_config_file = config_path + '/ci-' + config_file - #rruCheck = False - #result = re.search('rru', str(config_file)) - #if result is not None: - # rruCheck = True - ## Make a copy and adapt to EPC / UE IP addresses - #self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) - #self.command('sed -i -e \'s/CI_UE_IP_ADDR/' + self.UEIPAddress + '/\' ' + ci_full_config_file, '\$', 2); - # Launch UE with the modified config file self.command('source oaienv', '\$', 5) self.command('cd cmake_targets/lte_build_oai/build', '\$', 5) + result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args)) + # We may have to regenerate the .u* files + if result is None: + self.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf .u*', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S ../../../targets/bin/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5) + # Launch UE with the modified config file self.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5) @@ -640,27 +630,6 @@ class SSHConnection(): while (doLoop): loopCounter = loopCounter - 1 if (loopCounter == 0): - # In case of T tracer recording, we may need to kill it - #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) - #if result is not None: - # self.command('killall --signal SIGKILL record', '\$', 5) - self.close() - doLoop = False - logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m') - self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE') - self.CreateHtmlTabFooter(False) - ## In case of T tracer recording, we need to kill tshark on EPC side - #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) - #if result is not None: - # self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) - # logging.debug('\u001B[1m Stopping tshark \u001B[0m') - # self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) - # self.close() - # time.sleep(1) - # pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' - # copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') - # if (copyin_res == 0): - # self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') sys.exit(1) else: self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) @@ -817,8 +786,9 @@ class SSHConnection(): if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) self.CreateHtmlTabFooter(False) @@ -843,7 +813,7 @@ class SSHConnection(): return ping_time = re.findall("-c (\d+)",str(self.ping_args)) device_id = 'catm' - ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + str(moduleIPAddr) + ' 2>&1 | stdbuf -o0 tee -a ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) + ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + str(moduleIPAddr) + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) # TIMEOUT CASE if ping_status < 0: message = 'Ping with UE (' + str(moduleIPAddr) + ') crashed due to TIMEOUT!' @@ -956,8 +926,9 @@ class SSHConnection(): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow('N/A', 'KO', pStatus) self.AutoTerminateUEandeNB() @@ -1021,8 +992,9 @@ class SSHConnection(): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow('N/A', 'KO', pStatus) self.AutoTerminateUEandeNB() @@ -1085,8 +1057,9 @@ class SSHConnection(): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow('N/A', 'KO', pStatus) self.CreateHtmlTabFooter(False) @@ -1173,7 +1146,7 @@ class SSHConnection(): self.command('cd ' + self.EPCSourceCodePath, '\$', 5) self.command('cd scripts', '\$', 5) ping_time = re.findall("-c (\d+)",str(self.ping_args)) - ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee -a ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) + ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) # TIMEOUT CASE if ping_status < 0: message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!' @@ -1233,12 +1206,107 @@ class SSHConnection(): except: os.kill(os.getppid(),signal.SIGUSR1) + def PingNoS1(self): + check_eNB = True + check_OAI_UE = True + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) + if (pStatus < 0): + self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) + self.AutoTerminateUEandeNB() + self.CreateHtmlTabFooter(False) + sys.exit(1) + ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args)) + if ping_from_eNB is not None: + if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '': + Usage() + sys.exit('Insufficient Parameter') + else: + if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '': + Usage() + sys.exit('Insufficient Parameter') + try: + if ping_from_eNB is not None: + self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) + self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets/', '\$', 5) + else: + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath + '/cmake_targets/', '\$', 5) + self.command('cd cmake_targets', '\$', 5) + ping_time = re.findall("-c (\d+)",str(self.ping_args)) + ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '.log', '\$', int(ping_time[0])*1.5) + # TIMEOUT CASE + if ping_status < 0: + message = 'Ping with OAI UE crashed due to TIMEOUT!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + return + result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before)) + if result is None: + message = 'Packet Loss Not Found!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + return + packetloss = result.group('packetloss') + if float(packetloss) == 100: + message = 'Packet Loss is 100%' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + return + result = re.search('rtt min\/avg\/max\/mdev = (?P[0-9\.]+)\/(?P[0-9\.]+)\/(?P[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before)) + if result is None: + message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + return + rtt_min = result.group('rtt_min') + rtt_avg = result.group('rtt_avg') + rtt_max = result.group('rtt_max') + pal_msg = 'Packet Loss : ' + packetloss + '%' + min_msg = 'RTT(Min) : ' + rtt_min + ' ms' + avg_msg = 'RTT(Avg) : ' + rtt_avg + ' ms' + max_msg = 'RTT(Max) : ' + rtt_max + ' ms' + logging.debug('\u001B[1;37;44m OAI UE ping result \u001B[0m') + logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m') + qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + packetLossOK = True + if packetloss is not None: + if float(packetloss) > float(self.ping_packetloss_threshold): + qMsg += '\nPacket Loss too high' + logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m') + packetLossOK = False + elif float(packetloss) > 0: + qMsg += '\nPacket Loss is not 0%' + logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m') + self.close() + html_queue = SimpleQueue() + ip_addr = 'TBD' + html_cell = '
OAI UE ping result\n' + qMsg + '
' + html_queue.put(html_cell) + if packetLossOK: + self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue) + else: + self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) + + # copying on the EPC server for logCollection + if ping_from_eNB is not None: + copyin_res = self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.') + else: + copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.') + if (copyin_res == 0): + self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'ping_' + self.testCase_id + '.log', self.EPCSourceCodePath + '/scripts') + except: + os.kill(os.getppid(),signal.SIGUSR1) + def Ping(self): + result = re.search('noS1', str(self.Initialize_eNB_args)) + if result is not None: + self.PingNoS1() + return if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) self.CreateHtmlTabFooter(False) @@ -1599,7 +1667,7 @@ class SSHConnection(): time.sleep(0.5) self.command('rm -f iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) - iperf_status = self.command('stdbuf -o0 adb -s ' + device_id + ' shell "/data/local/tmp/iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + '" 2>&1 | stdbuf -o0 tee -a iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) + iperf_status = self.command('stdbuf -o0 adb -s ' + device_id + ' shell "/data/local/tmp/iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + '" 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) # TIMEOUT Case if iperf_status < 0: self.close() @@ -1679,12 +1747,12 @@ class SSHConnection(): self.command('rm -f iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) if (useIperf3): - self.command('stdbuf -o0 iperf3 -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) + self.command('stdbuf -o0 iperf3 -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) clientStatus = 0 self.Iperf_analyzeV3Output(lock, UE_IPAddress, device_id, statusQueue) else: - iperf_status = self.command('stdbuf -o0 iperf -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) + iperf_status = self.command('stdbuf -o0 iperf -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee iperf_' + self.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0) if iperf_status < 0: self.close() message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !' @@ -1710,12 +1778,118 @@ class SSHConnection(): except: os.kill(os.getppid(),signal.SIGUSR1) + def IperfNoS1(self): + check_eNB = True + check_OAI_UE = True + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) + if (pStatus < 0): + self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) + self.AutoTerminateUEandeNB() + self.CreateHtmlTabFooter(False) + sys.exit(1) + if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '': + Usage() + sys.exit('Insufficient Parameter') + server_on_enb = re.search('-R', str(self.iperf_args)) + if server_on_enb is not None: + iServerIPAddr = self.eNBIPAddress + iServerUser = self.eNBUserName + iServerPasswd = self.eNBPassword + iClientIPAddr = self.UEIPAddress + iClientUser = self.UEUserName + iClientPasswd = self.UEPassword + else: + iServerIPAddr = self.UEIPAddress + iServerUser = self.UEUserName + iServerPasswd = self.UEPassword + iClientIPAddr = self.eNBIPAddress + iClientUser = self.eNBUserName + iClientPasswd = self.eNBPassword + # Starting the iperf server + self.open(iServerIPAddr, iServerUser, iServerPasswd) + # args SHALL be "-c client -u any" + # -c 10.0.1.2 -u -b 1M -t 30 -i 1 -fm -B 10.0.1.1 + # -B 10.0.1.1 -u -s -i 1 -fm + server_options = re.sub('-u.*$', '-u -s -i 1 -fm', str(self.iperf_args)) + server_options = server_options.replace('-c','-B') + self.command('rm -f /tmp/tmp_iperf_server_' + self.testCase_id + '.log', '\$', 5) + self.command('echo $USER; nohup iperf ' + server_options + ' > /tmp/tmp_iperf_server_' + self.testCase_id + '.log 2>&1 &', iServerUser, 5) + time.sleep(0.5) + self.close() + + # Starting the iperf client + modified_options = self.Iperf_ComputeModifiedBW(0, 1) + modified_options = modified_options.replace('-R','') + iperf_time = self.Iperf_ComputeTime() + self.open(iClientIPAddr, iClientUser, iClientPasswd) + self.command('rm -f /tmp/tmp_iperf_' + self.testCase_id + '.log', '\$', 5) + iperf_status = self.command('stdbuf -o0 iperf ' + modified_options + ' 2>&1 | stdbuf -o0 tee /tmp/tmp_iperf_' + self.testCase_id + '.log', '\$', int(iperf_time)*5.0) + status_queue = SimpleQueue() + lock = Lock() + if iperf_status < 0: + message = 'iperf on OAI UE crashed due to TIMEOUT !' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + clientStatus = -2 + else: + clientStatus = self.Iperf_analyzeV2Output(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options) + self.close() + + # Stopping the iperf server + self.open(iServerIPAddr, iServerUser, iServerPasswd) + self.command('killall --signal SIGKILL iperf', '\$', 5) + time.sleep(0.5) + self.close() + if (clientStatus == -1): + if (os.path.isfile('iperf_server_' + self.testCase_id + '.log')): + os.remove('iperf_server_' + self.testCase_id + '.log') + self.copyin(iServerIPAddr, iServerUser, iServerPasswd, '/tmp/tmp_iperf_server_' + self.testCase_id + '.log', 'iperf_server_' + self.testCase_id + '_OAI-UE.log') + self.Iperf_analyzeV2Server(lock, '10.0.1.2', 'OAI-UE', status_queue, modified_options) + + # copying on the EPC server for logCollection + copyin_res = self.copyin(iServerIPAddr, iServerUser, iServerPasswd, '/tmp/tmp_iperf_server_' + self.testCase_id + '.log', 'iperf_server_' + self.testCase_id + '_OAI-UE.log') + if (copyin_res == 0): + self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_server_' + self.testCase_id + '_OAI-UE.log', self.EPCSourceCodePath + '/scripts') + copyin_res = self.copyin(iClientIPAddr, iClientUser, iClientPasswd, '/tmp/tmp_iperf_' + self.testCase_id + '.log', 'iperf_' + self.testCase_id + '_OAI-UE.log') + if (copyin_res == 0): + self.copyout(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, 'iperf_' + self.testCase_id + '_OAI-UE.log', self.EPCSourceCodePath + '/scripts') + iperf_noperf = False + if status_queue.empty(): + iperf_status = False + else: + iperf_status = True + html_queue = SimpleQueue() + while (not status_queue.empty()): + count = status_queue.get() + if (count < 0): + iperf_status = False + if (count > 0): + iperf_noperf = True + device_id = status_queue.get() + ip_addr = status_queue.get() + message = status_queue.get() + html_cell = '
UE (' + device_id + ')\nIP Address  : ' + ip_addr + '\n' + message + '
' + html_queue.put(html_cell) + if (iperf_noperf and iperf_status): + self.CreateHtmlTestRowQueue(self.iperf_args, 'PERF NOT MET', len(self.UEDevices), html_queue) + elif (iperf_status): + self.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) + else: + self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) + self.AutoTerminateUEandeNB() + self.CreateHtmlTabFooter(False) + sys.exit(1) + def Iperf(self): + result = re.search('noS1', str(self.Initialize_eNB_args)) + if result is not None: + self.IperfNoS1() + return if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() sys.exit('Insufficient Parameter') - initialize_eNB_flag = False - pStatus = self.CheckProcessExist(initialize_eNB_flag) + check_eNB = True + check_OAI_UE = False + pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE) if (pStatus < 0): self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) self.AutoTerminateUEandeNB() @@ -1772,26 +1946,37 @@ class SSHConnection(): self.CreateHtmlTabFooter(False) sys.exit(1) - def CheckProcessExist(self, initialize_eNB_flag): + def CheckProcessExist(self, check_eNB, check_OAI_UE): multi_jobs = [] status_queue = SimpleQueue() - p = Process(target = SSH.CheckHSSProcess, args = (status_queue,)) - p.daemon = True - p.start() - multi_jobs.append(p) - p = Process(target = SSH.CheckMMEProcess, args = (status_queue,)) - p.daemon = True - p.start() - multi_jobs.append(p) - p = Process(target = SSH.CheckSPGWProcess, args = (status_queue,)) - p.daemon = True - p.start() - multi_jobs.append(p) - if initialize_eNB_flag == False: + # in noS1 config, no need to check status from EPC + result = re.search('noS1', str(self.Initialize_eNB_args)) + if result is None: + p = Process(target = SSH.CheckHSSProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) + p = Process(target = SSH.CheckMMEProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) + p = Process(target = SSH.CheckSPGWProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) + else: + if (check_eNB == False) and (check_OAI_UE == False): + return 0 + if check_eNB: p = Process(target = SSH.CheckeNBProcess, args = (status_queue,)) p.daemon = True p.start() multi_jobs.append(p) + if check_OAI_UE: + p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) for job in multi_jobs: job.join() @@ -1810,29 +1995,8 @@ class SSHConnection(): logStatus = self.AnalyzeLogFile_eNB(self.eNBLogFile) if logStatus < 0: result = logStatus - return result - - def CheckOAIUEProcessExist(self, initialize_OAI_UE_flag): - multi_jobs = [] - status_queue = SimpleQueue() - if initialize_OAI_UE_flag == False: - p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,)) - p.daemon = True - p.start() - multi_jobs.append(p) - for job in multi_jobs: - job.join() - - if (status_queue.empty()): - return -15 - else: - result = 0 - while (not status_queue.empty()): - status = status_queue.get() - if (status < 0): - result = status if result == OAI_UE_PROCESS_FAILED: - fileCheck = re.search('enb_', str(self.UELogFile)) + fileCheck = re.search('ue_', str(self.UELogFile)) if fileCheck is not None: self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') logStatus = self.AnalyzeLogFile_UE(self.UELogFile) @@ -1842,8 +2006,8 @@ class SSHConnection(): def CheckOAIUEProcess(self, status_queue): try: - self.open(self.OAIUEIPAddress, self.OAIUEUserName, self.OAIUEPassword) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never lte-uesoftmodem', '\$', 5) + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) result = re.search('lte-uesoftmodem', str(self.ssh.before)) if result is None: logging.debug('\u001B[1;37;41m OAI UE Process Not Found! \u001B[0m') @@ -1857,7 +2021,7 @@ class SSHConnection(): def CheckeNBProcess(self, status_queue): try: self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never lte-softmodem', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) result = re.search('lte-softmodem', str(self.ssh.before)) if result is None: logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m') @@ -1871,7 +2035,7 @@ class SSHConnection(): def CheckHSSProcess(self, status_queue): try: self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never hss', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never hss | grep -v grep', '\$', 5) if re.match('OAI', self.EPCType, re.IGNORECASE): result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) else: @@ -1888,7 +2052,7 @@ class SSHConnection(): def CheckMMEProcess(self, status_queue): try: self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never mme', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never mme | grep -v grep', '\$', 5) if re.match('OAI', self.EPCType, re.IGNORECASE): result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) else: @@ -1906,10 +2070,10 @@ class SSHConnection(): try: self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) if re.match('OAI', self.EPCType, re.IGNORECASE): - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never spgw', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never spgw | grep -v grep', '\$', 5) result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) else: - self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never xGw', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never xGw | grep -v grep', '\$', 5) result = re.search('xGw', str(self.ssh.before)) if result is None: logging.debug('\u001B[1;37;41m SPGW Process Not Found! \u001B[0m') @@ -1924,6 +2088,7 @@ class SSHConnection(): if (not os.path.isfile('./' + eNBlogFile)): return -1 enb_log_file = open('./' + eNBlogFile, 'r') + exitSignalReceived = False foundAssertion = False msgAssertion = '' msgLine = 0 @@ -1954,17 +2119,20 @@ class SSHConnection(): self.htmleNBFailureMsg += ' Command line option(s) correctly applied ' + self.rruOptions + '\n\n' else: self.htmleNBFailureMsg += ' Command line option(s) NOT applied ' + self.rruOptions + '\n\n' - result = re.search('[Ss]egmentation [Ff]ault', str(line)) + result = re.search('Exiting OAI softmodem', str(line)) if result is not None: + exitSignalReceived = True + result = re.search('[Ss]egmentation [Ff]ault', str(line)) + if result is not None and not exitSignalReceived: foundSegFault = True result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundSegFault = True result = re.search('[Aa]ssertion', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundAssertion = True result = re.search('LLL', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundRealTimeIssue = True if foundAssertion and (msgLine < 3): msgLine += 1 @@ -2075,39 +2243,52 @@ class SSHConnection(): if (not os.path.isfile('./' + UElogFile)): return -1 ue_log_file = open('./' + UElogFile, 'r') + exitSignalReceived = False foundAssertion = False msgAssertion = '' msgLine = 0 foundSegFault = False foundRealTimeIssue = False - rlcDiscardBuffer = 0 - rachCanceledProcedure = 0 uciStatMsgCount = 0 - pdcpFailure = 0 - ulschFailure = 0 + pdcpDataReqFailedCount = 0 + badDciCount = 0 + rrcConnectionRecfgComplete = 0 no_cell_sync_found = False mib_found = False frequency_found = False + plmn_found = False self.htmlUEFailureMsg = '' for line in ue_log_file.readlines(): - result = re.search('[Ss]egmentation [Ff]ault', str(line)) + result = re.search('Exiting OAI softmodem', str(line)) if result is not None: + exitSignalReceived = True + result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line)) + if result is not None and not exitSignalReceived: foundSegFault = True result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundSegFault = True result = re.search('[Aa]ssertion', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundAssertion = True result = re.search('LLL', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: foundRealTimeIssue = True if foundAssertion and (msgLine < 3): msgLine += 1 msgAssertion += str(line) result = re.search('uci->stat', str(line)) - if result is not None: + if result is not None and not exitSignalReceived: uciStatMsgCount += 1 + result = re.search('PDCP data request failed', str(line)) + if result is not None and not exitSignalReceived: + pdcpDataReqFailedCount += 1 + result = re.search('bad DCI 1A', str(line)) + if result is not None and not exitSignalReceived: + badDciCount += 1 + result = re.search('Generating RRCConnectionReconfigurationComplete', str(line)) + if result is not None: + rrcConnectionRecfgComplete += 1 # No cell synchronization found, abandoning result = re.search('No cell synchronization found, abandoning', str(line)) if result is not None: @@ -2145,6 +2326,15 @@ class SSHConnection(): frequency_found = True except Exception as e: logging.error('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m') + result = re.search("PLMN MCC (?P\d{1,3}), MNC (?P\d{1,3}), TAC", str(line)) + if result is not None and (not plmn_found): + try: + mibMsg = 'PLMN MCC = ' + result.group('mcc') + ' MNC = ' + result.group('mnc') + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + plmn_found = True + except Exception as e: + logging.error('\033[91m' + "PLMN not found" + '\033[0m') result = re.search("Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line)) if result is not None: try: @@ -2180,32 +2370,30 @@ class SSHConnection(): except Exception as e: logging.error('\033[91m' + " AllowedMeasBandwidth not found" + '\033[0m') ue_log_file.close() + if rrcConnectionRecfgComplete > 0: + statMsg = 'UE connected to eNB (' + str(rrcConnectionRecfgComplete) + ' RRCConnectionReconfigurationComplete message(s) generated)' + logging.debug('\033[94m' + statMsg + '\033[0m') + self.htmlUEFailureMsg += statMsg + '\n' if uciStatMsgCount > 0: statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') self.htmlUEFailureMsg += statMsg + '\n' - if pdcpFailure > 0: - statMsg = 'UE showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)' + if pdcpDataReqFailedCount > 0: + statMsg = 'UE showed ' + str(pdcpDataReqFailedCount) + ' "PDCP data request failed" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') self.htmlUEFailureMsg += statMsg + '\n' - if ulschFailure > 0: - statMsg = 'UE showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' + if badDciCount > 0: + statMsg = 'UE showed ' + str(badDciCount) + ' "bad DCI 1A" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') self.htmlUEFailureMsg += statMsg + '\n' - if rachCanceledProcedure > 0: - rachMsg = 'UE cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' - logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rachMsg + '\n' if foundSegFault: logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') return ENB_PROCESS_SEG_FAULT if foundAssertion: - logging.debug('\u001B[1;37;43m UE ended with an assertion! \u001B[0m') - # removed for esthetics - #self.htmlUEFailureMsg += msgAssertion - self.htmlUEFailureMsg += 'UE ended with an assertion!\n' + logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m') + self.htmlUEFailureMsg += 'UE showed an assertion!\n' if not mib_found or not frequency_found: - return UE_PROCESS_ASSERTION + return OAI_UE_PROCESS_ASSERTION if foundRealTimeIssue: logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') self.htmlUEFailureMsg += 'UE faced real time issues!\n' @@ -2213,26 +2401,24 @@ class SSHConnection(): if no_cell_sync_found and not mib_found: logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m') self.htmlUEFailureMsg += 'UE could not synchronize!\n' - return UE_PROCESS_COULD_NOT_SYNC - if rlcDiscardBuffer > 0: - rlcMsg = 'UE RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)' - logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rlcMsg + '\n' - return ENB_PROCESS_REALTIME_ISSUE + return OAI_UE_PROCESS_COULD_NOT_SYNC return 0 def TerminateeNB(self): self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5) - self.command('echo ' + self.eNBPassword + ' | sudo -S daemon --name=enb' + str(self.eNB_instance) + '_daemon --stop', '\$', 5) - self.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) - self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGINT lte-softmodem || true', '\$', 5) - time.sleep(5) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep lte-softmodem', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) result = re.search('lte-softmodem', str(self.ssh.before)) if result is not None: - self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S daemon --name=enb' + str(self.eNB_instance) + '_daemon --stop', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGINT lte-softmodem || true', '\$', 5) time.sleep(5) + self.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) + result = re.search('lte-softmodem', str(self.ssh.before)) + if result is not None: + self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5) + time.sleep(2) + self.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) self.close() # If tracer options is on, stopping tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) @@ -2289,7 +2475,7 @@ class SSHConnection(): if re.match('OAI', self.EPCType, re.IGNORECASE): self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_hss oai_hss || true', '\$', 5) time.sleep(2) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep hss', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep hss | grep -v grep', '\$', 5) result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) if result is not None: self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_hss oai_hss || true', '\$', 5) @@ -2308,7 +2494,7 @@ class SSHConnection(): if re.match('OAI', self.EPCType, re.IGNORECASE): self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_mme mme || true', '\$', 5) time.sleep(2) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep mme', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep mme | grep -v grep', '\$', 5) result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) if result is not None: self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_mme mme || true', '\$', 5) @@ -2323,7 +2509,7 @@ class SSHConnection(): if re.match('OAI', self.EPCType, re.IGNORECASE): self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_spgw spgw || true', '\$', 5) time.sleep(2) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep spgw', '\$', 5) + self.command('stdbuf -o0 ps -aux | grep spgw | grep -v grep', '\$', 5) result = re.search('\/bin\/bash .\/run_', str(self.ssh.before)) if result is not None: self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_spgw spgw || true', '\$', 5) @@ -2364,82 +2550,72 @@ class SSHConnection(): def TerminateOAIUE(self): self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) - self.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5) - self.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) - self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5) - time.sleep(5) - self.command('stdbuf -o0 ps -aux | grep -v grep | grep lte-uesoftmodem', '\$', 5) + self.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) result = re.search('lte-uesoftmodem', str(self.ssh.before)) if result is not None: - self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5) time.sleep(5) + self.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5) + result = re.search('lte-uesoftmodem', str(self.ssh.before)) + if result is not None: + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5) + time.sleep(2) + self.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) self.close() - # If tracer options is on, stopping tshark on EPC side - result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + result = re.search('ue_', str(self.UELogFile)) if result is not None: - self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) - logging.debug('\u001B[1m Stopping tshark \u001B[0m') - self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) - time.sleep(1) - pcap_log_file = self.UELogFile.replace('.log', '_s1log.pcap') - self.command('echo ' + self.UEPassword + ' | sudo -S chmod 666 /tmp/' + pcap_log_file, '\$', 5) - self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, '/tmp/' + pcap_log_file, '.') - self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') - self.close() - logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') - self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) - self.command('cd ' + self.UESourceCodePath + '/common/utils/T/tracer/', '\$', 5) - raw_record_file = self.UELogFile.replace('.log', '_record.raw') - replay_log_file = self.UELogFile.replace('.log', '_replay.log') - extracted_txt_file = self.UELogFile.replace('.log', '_extracted_messages.txt') - extracted_log_file = self.UELogFile.replace('.log', '_extracted_messages.log') - self.command('./extract_config -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file, '\$', 5) - self.command('echo $USER; nohup ./replay -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', self.UEUserName, 5) - self.command('./textlog -d ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '\$', 5) - self.close() - self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '.') - logging.debug('\u001B[1m Analyzing UE replay logfile \u001B[0m') - logStatus = self.AnalyzeLogFile_UE(extracted_log_file) - self.CreateHtmlTestRow(html_queue, 'OK', ALL_PROCESSES_OK) - self.UELogFile = '' - else: - result = re.search('ue_', str(self.UELogFile)) + copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') + if (copyin_res == -1): + logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m') + self.htmlUEFailureMsg = 'Could not copy UE logfile to analyze it!' + self.CreateHtmlTestRow('N/A', 'KO', OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE') + self.UELogFile = '' + return + logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m') + logStatus = self.AnalyzeLogFile_UE(self.UELogFile) + result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args)) if result is not None: - copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') - if (copyin_res == -1): - logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m') - optionsMsg = '
Could not copy UE logfile to analyze it!
' - self.CreateHtmlTestRow(optionsMsg, 'KO', UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE') - self.UELogFile = '' - return - logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m') - logStatus = self.AnalyzeLogFile_UE(self.UELogFile) - if (logStatus < 0): - optionsMsg = '
Sniffing Unsuccessful\n'
-					optionsMsg += self.htmlUEFailureMsg
-					optionsMsg += '
' - self.CreateHtmlTestRow(optionsMsg, 'KO', logStatus, 'UE') + ueAction = 'Sniffing' + else: + ueAction = 'Connection' + if (logStatus < 0): + logging.debug('\u001B[1m' + ueAction + ' Failed \u001B[0m') + self.htmlUEFailureMsg = '' + ueAction + ' Failed\n' + self.htmlUEFailureMsg + self.CreateHtmlTestRow('N/A', 'KO', logStatus, 'UE') + # In case of sniffing on commercial eNBs we have random results + # Not an error then + if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'): + self.Initialize_OAI_UE_args = '' + self.AutoTerminateUEandeNB() self.CreateHtmlTabFooter(False) sys.exit(1) - else: - optionsMsg = '
Sniffing Successful\n'
-					optionsMsg += self.htmlUEFailureMsg
-					optionsMsg += '
' - self.CreateHtmlTestRow(optionsMsg, 'OK', ALL_PROCESSES_OK) - self.UELogFile = '' else: - self.CreateHtmlTestRow('
No Log File to analyze
', 'OK', ALL_PROCESSES_OK) + logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m') + self.htmlUEFailureMsg = '' + ueAction + ' Completed\n' + self.htmlUEFailureMsg + self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + self.UELogFile = '' + else: + self.htmlUEFailureMsg = 'No Log File to analyze!' + self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) def AutoTerminateUEandeNB(self): - self.testCase_id = 'AUTO-KILL-UE' - self.desc = 'Automatic Termination of UE' - self.ShowTestID() - self.TerminateUE() - self.testCase_id = 'AUTO-KILL-eNB' - self.desc = 'Automatic Termination of eNB' - self.ShowTestID() - self.eNB_instance = '0' - self.TerminateeNB() + if (self.ADBIPAddress != 'none'): + self.testCase_id = 'AUTO-KILL-UE' + self.desc = 'Automatic Termination of UE' + self.ShowTestID() + self.TerminateUE() + if (self.Initialize_OAI_UE_args != ''): + self.testCase_id = 'AUTO-KILL-UE' + self.desc = 'Automatic Termination of UE' + self.ShowTestID() + self.TerminateOAIUE() + if (self.Initialize_eNB_args != ''): + self.testCase_id = 'AUTO-KILL-eNB' + self.desc = 'Automatic Termination of eNB' + self.ShowTestID() + self.eNB_instance = '0' + self.TerminateeNB() def IdleSleep(self): time.sleep(self.idle_sleep_time) @@ -2541,8 +2717,8 @@ class SSHConnection(): self.command('echo ' + self.UEPassword + ' | sudo -S rm ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 5) self.close() - def RetrieveSystemVersion(self): - if self.eNBIPAddress == 'none': + def RetrieveSystemVersion(self, machine): + if self.eNBIPAddress == 'none' or self.UEIPAddress == 'none': self.OsVersion = 'Ubuntu 16.04.5 LTS' self.KernelVersion = '4.15.0-45-generic' self.UhdVersion = '3.13.0.1-0' @@ -2550,21 +2726,22 @@ class SSHConnection(): self.CpuNb = '4' self.CpuModel = 'Intel(R) Core(TM) i5-6200U' self.CpuMHz = '2399.996 MHz' - return - machine = None - if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '': - machine = 'eNB' - IPAddress = self.eNBIPAddress - UserName = self.eNBUserName - Password = self.eNBPassword - elif self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '': - machine = 'UE' - IPAddress = self.UEIPAddress - UserName = self.UEUserName - Password = self.UEPassword - if machine is None: - Usage() - sys.exit('Insufficient Parameter') + return 0 + if machine == 'eNB': + if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '': + IPAddress = self.eNBIPAddress + UserName = self.eNBUserName + Password = self.eNBPassword + else: + return -1 + if machine == 'UE': + if self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '': + IPAddress = self.UEIPAddress + UserName = self.UEUserName + Password = self.UEPassword + else: + return -1 + self.open(IPAddress, UserName, Password) self.command('lsb_release -a', '\$', 5) result = re.search('Description:\\\\t(?P[a-zA-Z0-9\-\_\.\ ]+)', str(self.ssh.before)) @@ -2581,7 +2758,7 @@ class SSHConnection(): if result is not None: self.UhdVersion = result.group('uhd_version') logging.debug('UHD Version is: ' + self.UhdVersion) - self.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 15) result = re.search('product: (?P[0-9A-Za-z]+)\\\\r\\\\n', str(self.ssh.before)) if result is not None: self.UsrpBoard = result.group('usrp_board') @@ -2689,6 +2866,10 @@ class SSHConnection(): self.htmlUEConnected = len(self.UEDevices) self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') self.htmlFile.write('

' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server

\n') + else: + self.UEDevices.append('OAI-UE') + self.htmlUEConnected = len(self.UEDevices) + self.htmlFile.write('

' + str(len(self.UEDevices)) + ' OAI UE(s) is(are) connected to CI bench

\n') self.htmlFile.write('
\n') self.htmlFile.write('