Jenkinsfile-gitlab

#!/bin/groovy
/*
 * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The OpenAirInterface Software Alliance licenses this file to You under
 * the OAI Public License, Version 1.1  (the "License"); you may not use this file
 * except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.openairinterface.org/?page_id=698
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *-------------------------------------------------------------------------------
 * For more information about the OpenAirInterface (OAI) Software Alliance:
 *      contact@openairinterface.org
 */

// Abstraction function to send social media messages:
// like on Slack or Mattermost
def sendSocialMediaMessage(pipeChannel, pipeColor, pipeMessage) {
    if (params.pipelineUsesSlack != null) {
        if (params.pipelineUsesSlack) {
            slackSend channel: pipeChannel, color: pipeColor, message: pipeMessage
        }
    }
}

def doRedHatBuild = false
def doFlexranCtrlTest = false

// Location of the executor node
def nodeExecutor = params.nodeExecutor

pipeline {
    agent {
        label nodeExecutor
    }
    options {
        disableConcurrentBuilds()
        timestamps()
        gitLabConnection('OAI GitLab')
        gitlabBuilds(builds: [
            "Build gNB-USRP",
            "Build nr-UE-USRP",
            "Build eNB-USRP",
            "Build basic-sim",
            "Build phy-sim",
            "Build eNB-ethernet",
            "Build UE-ethernet",
            "Analysis with cppcheck",
            "Test phy-sim",
            "Test basic-sim",
            "Test L1-sim",
            "Test RF-sim",
            "Test L2-sim",
            "Test-Mono-FDD-Band7",
            "Test-Mono-TDD-Band40",
            "Test-IF4p5-FDD-Band7",
            "Test-IF4p5-TDD-Band40",
            "Test-Mono-FDD-Band13-LTE-M",
            "Test-IF4p5-TDD-Band38-Multi-RRU",
            "Test-eNB-OAI-UE-FDD-Band7",
            "Test-Mono-FDD-Band13-X2-HO",
            "Test-TDD-Band78-gNB-NR-UE",
            "Test-OCP-FDD-Band7"
        ])
        ansiColor('xterm')
    }

    stages {
        stage ("Verify Parameters") {
            steps {
                script {
                    JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
                    JOB_TIMESTAMP = JOB_TIMESTAMP.trim()

                    echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
                    def allParametersPresent = true
                    if (env.TESTPLATFORM_OWNER) {
                        echo "Platform is ${env.TESTPLATFORM_OWNER}"
                    }

                    if (params.RedHatRemoteServer == null) {
                        allParametersPresent = false
                    }
                    if (params.RedHatRemoteCredentials == null) {
                        allParametersPresent = false
                    }
                    if (params.RedHatWorkingPath == null) {
                        allParametersPresent = false
                    }
                    if (allParametersPresent) {
                        echo "Performing Red Hat Build"
                        doRedHatBuild = true
                    } else {
                        doRedHatBuild = false
                    }
                    if (params.FlexRanRtcGitLabRepository_Credentials != null) {
                        doFlexranCtrlTest = true
                    }
                    if (fileExists("flexran")) {
                        sh "rm -Rf flexran > /dev/null 2>&1"
                    }
                }
            }
        }
        stage ("Verify Guidelines") {
            steps {
                echo "Git URL         is ${GIT_URL}"
                echo "GitLab Act      is ${env.gitlabActionType}"
                script {
                    if ("MERGE".equals(env.gitlabActionType)) {
                        // since a bit, in push events, gitlabUserEmail is not populated
                        gitCommitAuthorEmailAddr = env.gitlabUserEmail
                        echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
                        // GitLab-Jenkins plugin integration is lacking to perform the merge by itself
                        // Doing it manually --> it may have merge conflicts
                        sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
                        sh "zip -r -qq localZip.zip ."

                        // Running astyle options on the list of modified files by the merge request
                        // For the moment, there is no fail criteria. Just a notification of number of files that do not follow
                        sh "./ci-scripts/checkCodingFormattingRules.sh --src-branch ${env.gitlabSourceBranch} --target-branch ${env.gitlabTargetBranch}"
                        def res=readFile('./oai_rules_result.txt').trim();
                        if ("0".equals(res)) {
                            def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): All Changed files in Merge Request follow OAI Formatting Rules"
                            addGitLabMRComment comment: message
                        } else {
                            def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Some Changed files in Merge Request DO NOT follow OAI Formatting Rules"
                            addGitLabMRComment comment: message
                        }
                    } else {
                        echo "Git Branch      is ${GIT_BRANCH}"
                        echo "Git Commit      is ${GIT_COMMIT}"
                        // since a bit, in push events, gitlabUserEmail is not populated
                        gitCommitAuthorEmailAddr = sh returnStdout: true, script: 'git log -n1 --pretty=format:%ae ${GIT_COMMIT}'
                        gitCommitAuthorEmailAddr = gitCommitAuthorEmailAddr.trim()
                        echo "GitLab Usermail is ${gitCommitAuthorEmailAddr}"
                        sh "git log -n1 --pretty=format:\"%s\" > .git/CI_COMMIT_MSG"

                        sh "zip -r -qq localZip.zip ."
                        // Running astyle options on all C/H files in the repository
                        // For the moment, there is no fail criteria. Just a notification of number of files that do not follow
                        sh "./ci-scripts/checkCodingFormattingRules.sh"
                    }
                    if (doFlexranCtrlTest) {
                        sh "mkdir flexran"
                        dir ('flexran') {
                            withCredentials([
                                [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.FlexRanRtcGitLabRepository_Credentials}", usernameVariable: 'git_username', passwordVariable: 'git_password']
                                ]) {
                                sh "git clone https://${git_username}:${git_password}@gitlab.eurecom.fr/flexran/flexran-rtc.git . > ../git_clone.log 2>&1"
                                sh "git checkout develop >> ../git_clone.log 2>&1"
                            }
                            sh "zip -r -qq flexran.zip ."
                        }
                    }
                }
            }
            post {
                failure {
                    script {
                        def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Merge Conflicts -- Cannot perform CI"
                        addGitLabMRComment comment: message
                        currentBuild.result = 'FAILURE'
                    }
                }
            }
        }

        stage ("Start VM -- basic-sim") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Start VM -- gnb-usrp") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant gnb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Start VM -- nr-ue-usrp") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant nr-ue-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Start VM -- enb-usrp") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant enb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Start VM -- phy-sim") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }
        
        stage ("Start VM -- enb-ethernet") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant enb-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Start VM -- ue-ethernet") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant ue-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }
        
        stage ("Start VM -- cppcheck") {
            steps {
                timeout (time: 5, unit: 'MINUTES') {
                    sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant cppcheck --job-name ${JOB_NAME} --build-id ${BUILD_ID} --daemon"
                }
            }
        }

        stage ("Variant Builds") {
            parallel {
                stage ("Analysis with cppcheck") {
                    steps {
                        gitlabCommitStatus(name: "Analysis with cppcheck") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant cppcheck --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                            }
                        }
                    }
                }
                stage ("Build basic simulator") {
                    steps {
                        gitlabCommitStatus(name: "Build basic-sim") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build 5G gNB-USRP") {
                    steps {
                        gitlabCommitStatus(name: "Build gNB-USRP") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant gnb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build 5G NR-UE-USRP") {
                    steps {
                        gitlabCommitStatus(name: "Build nr-UE-USRP") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant nr-ue-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build eNB-USRP") {
                    steps {
                        gitlabCommitStatus(name: "Build eNB-USRP") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant enb-usrp --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build physical simulators") {
                    steps {
                        gitlabCommitStatus(name: "Build phy-sim") {
                            timeout (time: 20, unit: 'MINUTES') {
                               sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build eNB-ethernet") {
                    steps {
                        gitlabCommitStatus(name: "Build eNB-ethernet") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant enb-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build UE-ethernet") {
                    steps {
                        gitlabCommitStatus(name: "Build UE-ethernet") {
                            timeout (time: 20, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool wait --workspace $WORKSPACE --variant ue-ethernet --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                            }
                        }
                    }
                }
                stage ("Build eNB-USRP on Red Hat") {
                    when {
                        expression {doRedHatBuild}
                    }
                    steps {
                        gitlabCommitStatus(name: "Build eNB-USRP-RHE") {
                            script {
                                try {
                                    withCredentials([
                                        [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.RedHatRemoteCredentials}", usernameVariable: 'RH_Username', passwordVariable: 'RH_Password']
                                    ]) {
                                        timeout (time: 20, unit: 'MINUTES') {
                                            sh "./ci-scripts/buildOnRH.sh --workspace $WORKSPACE --job-name ${JOB_NAME} --build-id ${BUILD_ID} --remote-host ${params.RedHatRemoteServer} --remote-path ${params.RedHatWorkingPath} --remote-user-name ${RH_Username} --remote-password ${RH_Password}"
                                        }
                                    }
                                } catch (Exception e) {
                                    echo "Red Hat build failed but we could keep running pipeline if all ubuntu-based build passed"
                                }
                            }
                        }
                    }
                }
            }
            post {
                failure {
                    script {
                        currentBuild.result = 'FAILURE'
                    }
                }
                always {
                    script {
                        dir ('archives') {
                            if (fileExists('red_hat')) {
                                sh "zip -r -qq vm_build_logs.zip basic_sim enb_usrp phy_sim cppcheck enb_eth ue_eth gnb_usrp nr_ue_usrp red_hat"
                            } else {
                                sh "zip -r -qq vm_build_logs.zip basic_sim enb_usrp phy_sim cppcheck enb_eth ue_eth gnb_usrp nr_ue_usrp"
                            }
                        }
                        if(fileExists('archives/vm_build_logs.zip')) {
                            archiveArtifacts artifacts: 'archives/vm_build_logs.zip'
                        }
                        if ("MERGE".equals(env.gitlabActionType)) {
                            sh "./ci-scripts/oai-ci-vm-tool report-build --workspace $WORKSPACE --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger merge-request --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
                            // If the merge request has introduced more CPPCHECK errors or warnings, notifications in GitLab
                            if (fileExists('oai_cppcheck_added_errors.txt')) {
                                def ret=readFile('./oai_cppcheck_added_errors.txt').trim();
                                if ("0".equals(ret)) {
                                    echo "No added cppcheck warnings/errors in this merge request"
                                } else {
                                    def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Some modified files in Merge Request MAY have INTRODUCED up to " + ret + " CPPCHECK errors/warnings"
                                    addGitLabMRComment comment: message
                                }
                            }
                            // If the merge request has introduced compilation warnings, notifications in GitLab
                            sh "./ci-scripts/checkAddedWarnings.sh --src-branch ${env.gitlabSourceBranch} --target-branch ${env.gitlabTargetBranch}"
                            def res=readFile('./oai_warning_files.txt').trim();
                            if ("0".equals(res)) {
                                echo "No issues w/ warnings/errors in this merge request"
                            } else {
                                def fileList=readFile('./oai_warning_files_list.txt').trim();
                                def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Some modified files in Merge Request MAY have INTRODUCED WARNINGS (" + fileList + ")"
                                addGitLabMRComment comment: message
                            }
                        } else {
                            sh "./ci-scripts/oai-ci-vm-tool report-build --workspace $WORKSPACE --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger push --branch ${GIT_BRANCH} --commit ${GIT_COMMIT}"
                        }
                        if(fileExists('build_results.html')) {
                            sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' build_results.html"
                            archiveArtifacts artifacts: 'build_results.html'
                        }
                    }
                }
            }
        }

        stage ("Variant Tests") {
            parallel {
                stage ("VM-based tests") {
                    stages {
                        stage ("Test physical simulators") {
                            steps {
                                script {
                                    timeout (time: 90, unit: 'MINUTES') {
                                        try {
                                            gitlabCommitStatus(name: "Test phy-sim") {
                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                                            }
                                        } catch (Exception e) {
                                          currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Build Flexran Controller") {
                            when {
                                expression {doFlexranCtrlTest}
                            }
                            steps {
                                script {
                                    timeout (time: 20, unit: 'MINUTES') {
                                        try {
                                            sh "./ci-scripts/oai-ci-vm-tool build --workspace $WORKSPACE --variant flexran-rtc --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                                        } catch (Exception e) {
                                          currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Test basic simulator") {
                            steps {
                                script {
                                    timeout (time: 30, unit: 'MINUTES') {
                                        try {
                                            gitlabCommitStatus(name: "Test basic-sim") {
                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant basic-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                                            }
                                        } catch (Exception e) {
                                          currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Test L1 simulator") {
                            steps {
                                script {
                                    timeout (time: 30, unit: 'MINUTES') {
                                        try {
                                            gitlabCommitStatus(name: "Test L1-sim") {
                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant l1-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                                            }
                                        } catch (Exception e) {
                                            currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Test RF simulator") {
                            steps {
                                script {
                                    timeout (time: 30, unit: 'MINUTES') {
                                        try {
                                            gitlabCommitStatus(name: "Test RF-sim") {
                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant rf-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive"
                                            }
                                        } catch (Exception e) {
                                            currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Test L2 simulator") {
                            steps {
                                script {
                                    timeout (time: 30, unit: 'MINUTES') {
                                        try {
                                            gitlabCommitStatus(name: "Test L2-sim") {
                                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant l2-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                                            }
                                        } catch (Exception e) {
                                            currentBuild.result = 'FAILURE'
                                        }
                                    }
                                }
                            }
                        }
                        stage ("Destroy all Virtual Machines") {
                            steps {
                                sh "./ci-scripts/oai-ci-vm-tool destroy --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                            }
                        }
                    }
                }
                stage ("Test MONOLITHIC - FDD - Band 7 - B210") {
                    steps {
                        script {
                            triggerSlaveJob ('eNB-CI-FDD-Band7-B210', 'Test-Mono-FDD-Band7')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-FDD-Band7-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test MONOLITHIC - TDD - Band 40 - B210") {
                    steps {
                        script {
                            triggerSlaveJob ('eNB-CI-TDD-Band40-B210', 'Test-Mono-TDD-Band40')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-TDD-Band40-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test IF4p5 - FDD - Band 7 - B210") {
                    steps {
                        script {
                            sh "sleep 60"
                            triggerSlaveJob ('eNB-CI-IF4p5-FDD-Band7-B210', 'Test-IF4p5-FDD-Band7')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-IF4p5-FDD-Band7-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test IF4p5 - TDD - Band 40 - B210") {
                    steps {
                        script {
                            sh "sleep 60"
                            triggerSlaveJob ('eNB-CI-IF4p5-TDD-Band40-B210', 'Test-IF4p5-TDD-Band40')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-IF4p5-TDD-Band40-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test MONOLITHIC - FDD - Band 13 - B210") {
                    steps {
                        script {
                            sh "sleep 60"
                            triggerSlaveJob ('eNB-CI-MONO-FDD-Band13-B210', 'Test-Mono-FDD-Band13-LTE-M')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-MONO-FDD-Band13-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test X2 Handover - FDD - Band 13 - B210") {
                    steps {
                        script {
                            sh "sleep 60"
                            triggerSlaveJob ('eNB-CI-MONO-FDD-Band13-X2HO-B210', 'Test-Mono-FDD-Band13-X2-HO')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-MONO-FDD-Band13-X2HO-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test IF4p5 - TDD - Band 38 - B210 - MultiRRU") {
                    steps {
                        script {
                            triggerSlaveJob ('eNB-CI-IF4p5-TDD-Band38-MultiRRU-B210', 'Test-IF4p5-TDD-Band38-Multi-RRU')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-CI-IF4p5-TDD-Band38-MultiRRU-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test OAI UE - FDD - Band 20 - B200") {
                    steps {
                        script {
                            triggerSlaveJobNoGitLab ('UE-CI-FDD-Band20-B200')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('UE-CI-FDD-Band20-B200')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test OAI UE - OAI eNB - FDD - Band 7 - B200") {
                    steps {
                        script {
                            // Delayed trigger on slave job, so it is always the last one to run
                            sh "sleep 240"
                            triggerSlaveJob ('eNB-UE-CI-MONO-FDD-Band7-B200', 'Test-eNB-OAI-UE-FDD-Band7')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('eNB-UE-CI-MONO-FDD-Band7-B200')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test OAI NR UE - OAI gNB - TDD - Band 78 - N300") {
                    steps {
                        script {
                            triggerSlaveJob ('gNB-nrUE-MONO-TDD-Band78-N300', 'Test-TDD-Band78-gNB-NR-UE')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('gNB-nrUE-MONO-TDD-Band78-N300')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
                stage ("Test OAI OCP-eNB - FDD - Band 7 - B210") {
                    steps {
                        script {
                            triggerSlaveJob ('OCPeNB-FDD-Band7-B210', 'Test-OCP-FDD-Band7')
                        }
                    }
                    post {
                        always {
                            script {
                                finalizeSlaveJob('OCPeNB-FDD-Band7-B210')
                            }
                        }
                        failure {
                            script {
                                currentBuild.result = 'FAILURE'
                            }
                        }
                    }
                }
            }
            post {
                always {
                    script {
                        dir ('archives') {
                            sh "if [ -d basic_sim/test ] || [ -d phy_sim/test ] || [ -d l2_sim/test ]; then zip -r -qq vm_tests_logs.zip */test ; fi"
                        }
                        if(fileExists('archives/vm_tests_logs.zip')) {
                            archiveArtifacts artifacts: 'archives/vm_tests_logs.zip'
                            if ("MERGE".equals(env.gitlabActionType)) {
                                sh "./ci-scripts/oai-ci-vm-tool report-test --workspace $WORKSPACE --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger merge-request --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
                            } else {
                                sh "./ci-scripts/oai-ci-vm-tool report-test --workspace $WORKSPACE --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger push --branch ${GIT_BRANCH} --commit ${GIT_COMMIT}"
                            }
                            if(fileExists('test_simulator_results.html')) {
                                sh "sed -i -e 's#Build-ID: ${BUILD_ID}#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_simulator_results.html"
                                archiveArtifacts artifacts: 'test_simulator_results.html'
                            }
                        }
                    }
                }
            }
        }
    }
    post {
        always {
            script {
                // Stage destroy may not be run if error in previous stage
                sh "./ci-scripts/oai-ci-vm-tool destroy --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                emailext attachmentsPattern: '*results*.html',
                     body: '''Hi,

Here are attached HTML report files for $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!

Regards,
OAI CI Team''',
                     replyTo: 'no-reply@openairinterface.org',
                     subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
                     to: gitCommitAuthorEmailAddr

                if (fileExists('.git/CI_COMMIT_MSG')) {
                    sh "rm -f .git/CI_COMMIT_MSG"
                }
            }
        }
        success {
            script {
                def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): passed (" + BUILD_URL + ")"
                if ("MERGE".equals(env.gitlabActionType)) {
                    echo "This is a MERGE event"
                    addGitLabMRComment comment: message
                    def message2 = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): passed (" + BUILD_URL + ") -- MergeRequest #" + env.gitlabMergeRequestIid + " (" + env.gitlabMergeRequestTitle + ")"
                    sendSocialMediaMessage('ci-enb', 'good', message2)
                } else {
                    sendSocialMediaMessage('ci-enb', 'good', message)
                }
            }
        }
        failure {
            script {
                def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")"
                if ("MERGE".equals(env.gitlabActionType)) {
                    echo "This is a MERGE event"
                    addGitLabMRComment comment: message
                    def message2 = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ") -- MergeRequest #" + env.gitlabMergeRequestIid + " (" + env.gitlabMergeRequestTitle + ")"
                    sendSocialMediaMessage('ci-enb', 'danger', message2)
                } else {
                    sendSocialMediaMessage('ci-enb', 'danger', message)
                }
            }
        }
    }
}

// ----  Slave Job functions

def triggerSlaveJob (jobName, gitlabStatusName) {
    // Workaround for the "cancelled" GitLab pipeline notification
    // The slave job is triggered with the propagate false so the following commands are executed
    // Its status is now PASS/SUCCESS from a stage pipeline point of view
    // localStatus variable MUST be analyzed to properly assess the status
    localStatus = build job: jobName,
        parameters: [
            string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
            string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)),
            string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)),
            booleanParam(name: 'eNB_mergeRequest', value: "MERGE".equals(env.gitlabActionType)),
            string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch))
        ], propagate: false
    localResult = localStatus.getResult()
    echo "${jobName} Slave Job status is ${localResult}"
    gitlabCommitStatus(name: gitlabStatusName) {
        if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
           echo "${jobName} Slave Job is OK"
        } else {
           echo "${jobName} Slave Job is KO"
           sh "ci-scripts/fail.sh"
        }
    }
}

def triggerSlaveJobNoGitLab (jobName) {
    // Workaround for the "cancelled" GitLab pipeline notification
    // The slave job is triggered with the propagate false so the following commands are executed
    // Its status is now PASS/SUCCESS from a stage pipeline point of view
    // localStatus variable MUST be analyzed to properly assess the status
    localStatus = build job: jobName,
        parameters: [
            string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
            string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)),
            string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)),
            booleanParam(name: 'eNB_mergeRequest', value: "MERGE".equals(env.gitlabActionType)),
            string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch))
        ], propagate: false
    localResult = localStatus.getResult()
    echo "${jobName} Slave Job status is ${localResult}"
    if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
       echo "${jobName} Slave Job is OK"
    } else {
       echo "${jobName} Slave Job is KO"
       sh "ci-scripts/fail.sh"
    }
}

def finalizeSlaveJob(jobName) {
    // In case of any non-success, we are retrieving the HTML report of the last completed
    // slave job. The only drop-back is that we may retrieve the HTML report of a previous build
    fileName = "test_results-${jobName}.html"
    if (!fileExists(fileName)) {
        copyArtifacts(projectName: jobName,
            filter: 'test_results*.html',
            selector: lastCompleted())
        if (fileExists(fileName)) {
            sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' ${fileName}"
            archiveArtifacts artifacts: fileName
        }
    }
}