Commit 2858c7cf authored by ismail's avatar ismail
Browse files

CI: bracket testing added


Signed-off-by: default avatarismail <mohammed.ismail@openairinterface.org>
parent 52ed32fc
#!/bin/groovy
/*
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The OpenAirInterface Software Alliance licenses this file to You under
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
* except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.openairinterface.org/?page_id=698
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*-------------------------------------------------------------------------------
* For more information about the OpenAirInterface (OAI) Software Alliance:
* contact@openairinterface.org
*/
//-------------------------------------------------------------------------------
// Location of the CN executor node
def cn_ci_host = params.Host_CN_CI_Server
// for lock
def cn_ci_resource = params.DockerContainers
def ds_tester_ci_resource = params.DsTester
// Location of the 2nd CN executor
def new_host_flag = false
def new_host = ""
def new_host_user = ""
// Location of the CN tester
def dsT_host_flag = false
def dsT_host = ""
def dsT_host_user = ""
def dsT_host_ip_addr = ""
// Flags
def scmEvent = false
def upstreamEvent = false
def deployed = true
// Default tag --> could be passed on by upstream job or by PR content
def nrfTag = params.nrfTag
//-------------------------------------------------------------------------------
// Pipeline start
pipeline {
agent {
label cn_ci_host
}
options {
disableConcurrentBuilds()
timestamps()
ansiColor('xterm')
lock(cn_ci_resource)
}
stages {
stage ('Verify Parameters') {
steps {
script {
echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
if (params.Host_CN_CI_2nd_Server_Flag != null) {
new_host_flag = params.Host_CN_CI_2nd_Server_Flag
if (new_host_flag) {
new_host = params.Host_CN_CI_2nd_Server
new_host_user = params.Host_CN_CI_2nd_Server_Login
echo "1st Node is ${NODE_NAME}"
echo "2nd Node is ${new_host}"
} else {
echo "Node is ${NODE_NAME}"
}
} else {
echo "Node is ${NODE_NAME}"
}
if (params.DS_Tester_Server_Flag != null) {
dsT_host_flag = params.DS_Tester_Server_Flag
if (dsT_host_flag) {
def allParametersPresent = true
if (params.DS_Tester_Server_Name == null) {
allParametersPresent = false
} else {
dsT_host = params.DS_Tester_Server_Name
}
if (params.DS_Tester_Server_Login == null) {
allParametersPresent = false
} else {
dsT_host_user = params.DS_Tester_Server_Login
}
if (params.DS_Tester_Server_IP_Addr == null) {
allParametersPresent = false
} else {
dsT_host_ip_addr = params.DS_Tester_Server_IP_Addr
}
if (allParametersPresent) {
echo "DS Tester is on ${dsT_host}"
} else {
echo "Some DS Tester parameters are missing!"
sh "./ci-scripts/fail.sh"
}
}
}
// Find out the cause of the trigger
for (cause in currentBuild.getBuildCauses()) {
if (cause.toString() ==~ /.*UpstreamCause.*/) {
upstreamEvent = true
//} else {
// scmEvent = true
}
}
if (upstreamEvent) {
if (params.NRF_TAG != null) {
nrfTag = params.NRF_TAG
echo "Upstream Job passed NRF_TAG to use: ${nrfTag}"
}
sh "git clean -x -d -f > /dev/null 2>&1"
sh "git fetch --prune > /dev/null 2>&1"
}
sh "mkdir -p archives DS-TEST-RESULTS"
// Verify that the images are available
try {
sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log'
sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
} catch (Exception e) {
error "OAI NRF Image tag to test does not exist!"
}
}
}
}
stage ('Deploy NRF-U18') {
steps {
script {
echo '\u2705 \u001B[32mDeploy NRF using Docker-Compose\u001B[0m'
// Prepare all needed files for docker-compose
// First put all correct tags to test
sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
dir('ci-scripts/dsTesterDockerCompose') {
sh 'docker-compose up -d > ../../archives/compose_nrf_up.log 2>&1'
sh 'sleep 60'
// Do a check on number of healthy containers
// 1 == nrf
ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
ret = ret.trim()
if (ret != '1') {
error "Deployment went wrong!"
}
}
}
}
post {
always {
script {
// Do docker logs to recover the configuration results
try {
sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf > archives/nrf_config.log'
} catch (Exception e) {
sh 'echo "STATUS: KO" >> archives/nrf_config.log'
}
}
}
success {
script {
sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log'
sh 'python3 ./ci-scripts/routeCheck.py --mode=Add --userName=' + dsT_host_user + ' --hostName=' + dsT_host
}
}
unsuccessful {
script {
sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log'
deployed = false
}
}
}
}
stage ('Bracket Testing with dsTester') {
when { expression {dsT_host_flag} }
steps {
lock (ds_tester_ci_resource) {
script {
echo '\u2705 \u001B[32mTesting with DS Tester\u001B[0m'
if (fileExists("dstester")) {
sh "rm -Rf dstester > /dev/null 2>&1"
}
sh "mkdir -p dstester"
dir ('dstester') {
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.dsTesterGitLabRepository_Credentials}", usernameVariable: 'git_username', passwordVariable: 'git_token']
]) {
sh "git clone https://${git_username}:${git_token}@github.com/OPENAIRINTERFACE/chasseur.git . > ../git_clone.log 2>&1"
sh "git checkout edge >> ../git_clone.log 2>&1"
dir ('jenkins') {
try {
sh "python3 ./dogmatix-agent.py -f ./suits/dc/nrf-bracket.yaml -d true | tee ../../DS-TEST-RESULTS/dsTester_Summary.txt"
} catch (Exception e) {
currentBuild.result = 'FAILURE'
echo "dsTester Running FAILED"
}
}
}
}
sh "python3 ./ci-scripts/toCheckDSTesterResult.py"
}
}
}
post {
always {
script {
// Copy the pcap and log from the container
sh "mkdir -p archives/pcaps archives/logs"
try {
sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap'
sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log'
} catch (Exception e) {
sh 'echo "Error in copying pcap & log from the container"'
}
}
}
}
}
stage ('Undeploy NRF') {
steps {
script {
echo '\u2705 \u001B[32mUn-Deploy NRF\u001B[0m'
sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
dir('ci-scripts/dsTesterDockerCompose') {
sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1'
}
}
}
}
}
post {
always {
script {
// Get logs if deployment fails
if (deployed != true) {
sh 'mkdir -p archives/logs'
sh 'docker logs cicd-oai-nrf > archives/logs/oai_nrf.log'
}
// Remove any leftover containers/networks
sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
dir('ci-scripts/dsTesterDockerCompose') {
sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1'
}
// Generating the HTML report
sh 'python3 ./ci-scripts/bracketTestHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
// Zipping all archived log files
sh "zip -r -qq cn5g_nrf_docker_bt_logs.zip archives DS-TEST-RESULTS"
if (fileExists('cn5g_nrf_docker_bt_logs.zip')) {
archiveArtifacts artifacts: 'cn5g_nrf_docker_bt_logs.zip'
}
if (fileExists('test_results_oai_nrf_bt.html')) {
archiveArtifacts artifacts: 'test_results_oai_nrf_bt.html'
}
}
}
}
}
......@@ -338,6 +338,35 @@ pipeline {
}
}
}
stage ('Bracket Testing with DsTester') {
steps {
script {
gitlabCommitStatus(name: "Bracket Test with DsTester") {
localStatus = build job: params.NRF_BT_PipelineName,
parameters: [
string(name: 'NRF_TAG', value: String.valueOf(nrf_tag))
], propagate: false
localResult = localStatus.getResult()
if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Bracket Test Job is OK"
} else {
echo "Bracket Test Job is KO"
sh "ci-scripts/fail.sh"
}
}
}
}
post {
always {
script {
copyArtifacts(projectName: params.AMF_BT_PipelineName,
filter: '*_results_oai_nrf_bt.html',
selector: lastCompleted())
}
}
}
}
stage ('Testing in CN-5G-FED environment') {
steps {
script {
......@@ -439,6 +468,10 @@ pipeline {
sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_cn5g.html"
sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_cn5g.html"
}
if (fileExists('test_results_oai_nrf_bt.html')) {
sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_nrf_bt.html"
sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_nrf_bt.html"
}
} else {
sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
}
......@@ -451,6 +484,10 @@ pipeline {
sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_cn5g.html"
archiveArtifacts artifacts: '*_results_oai_cn5g.html'
}
if (fileExists('test_results_oai_nrf_bt.html')) {
sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_nrf_bt.html"
archiveArtifacts artifacts: '*_results_oai_nrf_bt.html'
}
// Sending email to commiter
if (params.sendToCommitterEmail != null) {
......
version: '3.8'
services:
cicd_oai_nrf:
container_name: cicd-oai-nrf
image: oai-nrf:NRF_IMAGE_TAG
ports:
- 80
- 9090
command: >
bash -c "nohup tshark -i eth0 -w /tmp/nrf.pcap 2>&1 > /dev/null &
/openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o | tee /tmp/nrf.log 2>&1
"
cap_add:
- NET_ADMIN
environment:
- TZ=Europe/Paris
- NRF_INTERFACE_NAME_FOR_SBI=eth0
- NRF_INTERFACE_PORT_FOR_SBI=80
- NRF_INTERFACE_HTTP2_PORT_FOR_SBI=9090
- NRF_API_VERSION=v1
- INSTANCE=0
- PID_DIRECTORY=/var/run
networks:
cicd_public_net:
ipv4_address: 192.168.61.195
volumes:
- ./nrf-healthy-check.sh:/openair-nrf/bin/nrf-healthy-check.sh
healthcheck:
test: /bin/bash -c "/openair-nrf/bin/nrf-healthy-check.sh"
interval: 10s
timeout: 5s
retries: 5
networks:
cicd_public_net:
name: cicd-oai-public-net
driver: bridge
ipam:
config:
- subnet: 192.168.61.192/26
#!/bin/bash
STATUS=0
RESULT=$(ps aux | grep -v nohup || true)
SUB='/openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o'
if [[ $RESULT =~ $SUB ]]; then
STATUS=0
else
STATUS=-1
fi
exit $STATUS
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
import os
import re
import sys
import subprocess
import yaml
class IpRouteCheck():
def __init__(self):
self.mode = ''
self.userName = ''
self.hostName = ''
self.subnet = ''
self.gatwayIp = ''
self.interfaceName = ''
def getSubnet(self):
cmd = "egrep 'subnet' ci-scripts/dsTesterDockerCompose/docker-compose.yml"
ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
if ret.returncode == 0:
if ret.stdout is not None:
result = re.search("(?:[0-9]{1,3}[.]){3}[0-9]{1,3}/[0-9]{1,2}", ret.stdout.strip())
if result is not None:
self.subnet = result.group(0)
#print("found subnet:", self.subnet)
else:
print("subnet not found in docker compose")
sys.exit(-1)
else:
print("docker-compose file not found")
sys.exit(-1)
def getGatwayIp(self):
cmd = "ifconfig | grep 192.168.18"
ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
if ret.returncode == 0:
result = re.search("inet (?:[0-9]{1,3}[.]){3}[0-9]{1,3}", ret.stdout.strip())
self.gatwayIp = result.group(0)
#print("Gatway IP Address:", self.gatwayIp)
else:
print("No Gatway IP address starts with 196.168.18")
sys.exit(-1)
def routeCheck(self):
# Get the interface name
cmd = "ip route | grep 192.168.18.0/24 | awk {'print $3'}"
ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
#print('interface name:', ret.stdout.strip())
self.interfaceName = ret.stdout.strip()
# Check whether or not ip route exist
cmd = f'ip route | grep -c "{self.subnet}"'
ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
if ret.stdout is not None:
if ret.stdout.strip() == '1':
#print('Route exits')
if IPRC.mode == 'Delete':
IPRC.routeDel()
else:
sys.exit(0)
else:
#print("Route not found")
if IPRC.mode == 'Add':
IPRC.routeAdd()
else:
sys.exit(0)
def routeAdd(self):
# Add the route
cmd = f"sudo ip route add {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
print("Added ip route")
def routeDel(self):
# Delete the route
cmd = f"sudo ip route del {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
print("Deleted ip route")
def Usage():
print('----------------------------------------------------------------------------------------------------------------------')
print('routecheck.py')
print(' Add and Delete the ip route on the Server.')
print('----------------------------------------------------------------------------------------------------------------------')
print('Usage: python3 routecheck.py [options]')
print(' --help Show this help.')
print('---------------------------------------------------------------------------------------------- Mandatory Options -----')
print(' --mode=[Add/Delete]')
print(' --userName=[server userName where to add/delete]')
print(' --hostName=[server hostName where to add/delete]')
print('------------------------------------------------------------------------------------------------- Actions Syntax -----')
print('python3 routeCheck.py --mode=Add [Mandatory Options]')
print('python3 routeCheck.py --mode=Delete [Mandatory Options]')
#--------------------------------------------------------------------------------------------------------
#
# Start of main
#
#--------------------------------------------------------------------------------------------------------
argvs = sys.argv
IPRC = IpRouteCheck()
while len(argvs) > 1:
myArgv = argvs.pop(1)
if re.match('^\-\-help$', myArgv, re.IGNORECASE):
Usage()
sys.exit(0)
elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
IPRC.mode = matchReg.group(1)
elif re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE)
IPRC.userName = matchReg.group(1)
elif re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE)
IPRC.hostName = matchReg.group(1)
else:
sys.exit('Invalid Parameter: ' + myArgv)
if IPRC.mode == '' or IPRC.userName == '' or IPRC.hostName == '':
sys.exit('Missing Parameter in job description')
IPRC.getSubnet()
IPRC.getGatwayIp()
IPRC.routeCheck()
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
import re
import sys
import subprocess
import yaml
import os
import time
locexist = False
cwd = os.getcwd()
try:
with open(cwd + '/DS-TEST-RESULTS/dsTester_Summary.txt') as f:
for line in f:
if re.search('Result file is available here', str(line)):
result = re.search('(?:\/.+?\/)(.+?)(?:\/.+)', str(line))
if result:
result1 = re.search('^(.*/)([^/]*)$', str(result.group(0)))
subprocess.check_output(f'cp {result1.group(1)}* DS-TEST-RESULTS/', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
result2 = re.search('.*\/(.+)$', str(result.group(0)))
filename = result2.group(1)
locexist = True
except IOError:
sys.exit("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt")
if locexist:
try:
with open(cwd + f'/DS-TEST-RESULTS/{filename}') as f:
data = yaml.load(f)
if data["final-result"] == 'fail':
sys.exit('DsTester final result FAILED')
except IOError:
sys.exit(f'File not accessible to check DSTester result: DS-TEST-RESULTS/{filename}')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment