Skip to content
Snippets Groups Projects
Commit a75f2a82 authored by ismail's avatar ismail
Browse files

CI: updated version of script for CI on Cluster 


Signed-off-by: default avatarismail <mohammed.ismail@openairinterface.org>
parent 6473621a
No related branches found
No related tags found
1 merge request!12CI pipeline for OC deployment and testing with dsTester
Hide whitespace changes
Inline Side-by-side
ci-scripts/Jenkinsfile-GitLab-Helm
+ 21
− 99
View file @ a75f2a82
......@@ -82,12 +82,6 @@ pipeline {
JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
def allParametersPresent = true
if (params.remote_IPAddress == null) {
allParametersPresent = false
}
if (params.remote_Credentials == null) {
allParametersPresent = false
}
if (params.OC_Credentials == null) {
allParametersPresent = false
}
......@@ -131,16 +125,17 @@ pipeline {
}
}
// Clean workspace and prepare artifacts location
sh "git clean -x -d -f > /dev/null 2>&1"
sh "mkdir -p archives DS-TEST-RESULTS"
// Find out the cause of the trigger
for (cause in currentBuild.getBuildCauses()) {
if (cause.toString() ==~ /.*UpstreamCause.*/) {
upstreamEvent = true
//} else {
// scmEvent = true
}
}
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.remote_Credentials}", usernameVariable: 'remote_Username', passwordVariable: 'remote_Password'],
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.OC_Credentials}", usernameVariable: 'OC_Username', passwordVariable: 'OC_Password']
]) {
if (upstreamEvent) {
......@@ -159,36 +154,6 @@ pipeline {
sh "git clean -x -d -f > /dev/null 2>&1"
sh "git fetch --prune > /dev/null 2>&1"
sh 'git checkout -f ' + upstreamTagToUse
sh "zip -r -qq oai-cn5g-fed.zip .git"
sh "mkdir -p archives DS-TEST-RESULTS"
// Prepare the workspace in the remote server
copyTo2ndServer('oai-cn5g-fed.zip', true, ${remote_Username}, ${params.remote_IPAddress})
myShCmd('git clean -x -d -f > /dev/null 2>&1', true, ${remote_Username}, ${params.remote_IPAddress})
myShCmd('mkdir -p archives DS-TEST-RESULTS', true, ${remote_Username}, ${params.remote_IPAddress})
}
if (scmEvent) {
sh "git clean -x -d -f > /dev/null 2>&1"
if ("MERGE".equals(env.gitlabActionType)) {
sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
}
sh "zip -r -qq oai-cn5g-fed.zip .git"
sh "mkdir -p archives DS-TEST-RESULTS"
// Prepare the workspace in remote server
copyTo2ndServer('oai-cn5g-fed.zip', true, ${remote_Username}, ${params.remote_IPAddress})
myShCmd('git clean -x -d -f > /dev/null 2>&1', true, ${remote_Username}, ${params.remote_IPAddress})
if ("MERGE".equals(env.gitlabActionType)) {
myShCmd("./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}", new_host_flag, new_host_user, new_host)
}
myShCmd('mkdir -p archives DS-TEST-RESULTS', true, ${remote_Username}, ${params.remote_IPAddress})
}
if ((!upstreamEvent) && (!scmEvent)) {
sh "git clean -x -d -f > /dev/null 2>&1"
sh "zip -r -qq oai-cn5g-fed.zip .git"
sh "mkdir -p archives DS-TEST-RESULTS"
// Prepare the workspace in the remote server
copyTo2ndServer('oai-cn5g-fed.zip', true, remote_Username, params.remote_IPAddress)
myShCmd('git clean -x -d -f > /dev/null 2>&1', true, remote_Username, params.remote_IPAddress)
myShCmd('mkdir -p archives DS-TEST-RESULTS', true, remote_Username, params.remote_IPAddress)
}
imageTags = "mysql:5.7.30,oai-nrf:${nrfTag},oai-amf:${amfTag},oai-smf:${smfTag},oai-spgwu-tiny:${spgwuTag}"
}
......@@ -200,12 +165,9 @@ pipeline {
script {
echo '\u2705 \u001B[32mDeploy CN5G on Cluster\u001B[0m'
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.remote_Credentials}", usernameVariable: 'remote_Username', passwordVariable: 'remote_Password'],
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.OC_Credentials}", usernameVariable: 'OC_Username', passwordVariable: 'OC_Password']
]) {
dir('ci-scripts') {
sh "python3 helmDeploy.py --mode=Deploy --remoteIPAdd=${params.remote_IPAddress} --remoteUserName=${remote_Username} --remotePassword=${remote_Password} --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
}
sh "python3 ci-scripts/helmDeploy.py --mode=Deploy --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
}
}
}
......@@ -230,7 +192,7 @@ pipeline {
sh "python3 jenkins/dogmatix-agent.py -f jenkins/suit.yaml | tee ../DS-TEST-RESULTS/dsTester_Summary.txt"
} catch (Exception e) {
currentBuild.result = 'FAILURE'
echo "dsTester FAILED"
echo "dsTester Running FAILED"
}
}
}
......@@ -244,17 +206,14 @@ pipeline {
script {
echo '\u2705 \u001B[32mUnDeploy CN5G on Cluster\u001B[0m'
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.remote_Credentials}", usernameVariable: 'remote_Username', passwordVariable: 'remote_Password'],
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.OC_Credentials}", usernameVariable: 'OC_Username', passwordVariable: 'OC_Password']
]) {
try {
sh "python3 ci-scripts/helmDeploy.py --mode=GetLogs --remoteIPAdd=${params.remote_IPAddress} --remoteUserName=${remote_Username} --remotePassword=${remote_Password} --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
sh "python3 ci-scripts/helmDeploy.py --mode=GetLogs --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
} catch (Exception e) {
echo "Unable to collect the logs, configs, pcaps"
}
dir('ci-scripts') {
sh "python3 helmDeploy.py --mode=UnDeploy --remoteIPAdd=${params.remote_IPAddress} --remoteUserName=${remote_Username} --remotePassword=${remote_Password} --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
}
sh "python3 ci-scripts/helmDeploy.py --mode=UnDeploy --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
}
}
}
......@@ -262,63 +221,26 @@ pipeline {
}
post {
always {
script {
script {
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.OC_Credentials}", usernameVariable: 'OC_Username', passwordVariable: 'OC_Password']
]) {
// Remove any leftover containers/networks
sh "python3 ci-scripts/helmDeploy.py --mode=UnDeploy --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName} --imageTags=${imageTags}"
}
// Generating the HTML report
sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport1.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
// Zipping all archived log files
sh "zip -r -qq cn5g_fed_cluster_logs.zip archives DS-TEST-RESULTS"
sh "rm -rf archives DS-TEST-RESULTS"
if (fileExists('cn5g_fed_cluster_logs.zip')) {
archiveArtifacts artifacts: 'cn5g_fed_cluster_logs.zip'
}
if (fileExists('test_results_oai_cn5g.html')) {
archiveArtifacts artifacts: 'test_results_oai_cn5g.html'
}
}
}
}
}
// Functions
def copyTo2ndServer(filename, flag, user, host) {
if (flag) {
if ("oai-cn5g-fed.zip".equals(filename)) {
sh "ssh ${user}@${host} 'rm -rf /tmp/CI-CN5G-FED-RHEL8'"
sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN5G-FED-RHEL8'"
}
sh "scp ${filename} ${user}@${host}:/tmp/CI-CN5G-FED-RHEL8"
if ("oai-cn5g-fed.zip".equals(filename)) {
sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED-RHEL8 && unzip -qq oai-cn5g-fed.zip && rm oai-cn5g-fed.zip'"
sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED-RHEL8 && git checkout -f ${GIT_COMMIT}'"
sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED-RHEL8 && git log -n1'"
}
}
}
def copyFrom2ndServer(filename, target, flag, user, host) {
if (flag) {
sh "scp ${user}@${host}:/tmp/CI-CN5G-FED-RHEL8/${filename} ${target}"
}
}
def myShCmd(cmd, flag, user, host) {
if (flag) {
sh "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN5G-FED-RHEL8 && ${cmd}'"
} else {
sh "${cmd}"
}
}
def myShCmdWithLog(cmd, logFile, flag, user, host) {
if (flag) {
sh "ssh -t -t ${user}@${host} 'export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && ${cmd}' > ${logFile} 2>&1"
} else {
sh "${cmd} > ${logFile} 2>&1"
}
}
def myShRetCmd(cmd, flag, user, host) {
if (flag) {
ret = sh returnStdout: true, script: "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN5G-FED-RHEL8 && ${cmd}'"
} else {
ret = sh returnStdout: true, script: "${cmd}"
}
ret = ret.trim()
return ret
}
ci-scripts/helmDeploy.py
+ 101
− 135
View file @ a75f2a82
......@@ -31,7 +31,6 @@
# Import
#-----------------------------------------------------------
import logging
import sshconnection as SSH
import html
import os
import re
......@@ -46,13 +45,9 @@ logging.basicConfig(
class ClusterDeploy:
def __init__(self):
self.remoteIPAdd = ""
self.remoteUserName = ""
self.remotePassword = ""
self.OCUserName = ""
self.OCPassword = ""
self.OCProjectName = ""
self.sourceCodePath = "/tmp/CI-CN5G-FED-RHEL8"
self.imageTags = ""
self.mode = ""
......@@ -61,21 +56,14 @@ class ClusterDeploy:
#-----------------$
def Deploy_5gcn(self):
lIpAddr = self.remoteIPAdd
lUserName = self.remoteUserName
lPassWord = self.remotePassword
lSourcePath = self.sourceCodePath
ocUserName = self.OCUserName
ocPassword = self.OCPassword
ocProjectName = self.OCProjectName
limageTags = self.imageTags
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '' or ocUserName == '' or ocPassword == '' or ocProjectName == '' or limageTags == '':
if ocUserName == '' or ocPassword == '' or ocProjectName == '' or limageTags == '':
sys.exit('Insufficient Parameter')
logging.debug('Running on server: ' + lIpAddr)
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
mySSH.command('cd ' + lSourcePath, '\$', 5)
logging.debug('\u001B[1m Checking all 5GCN component IMAGES are pre existing and correct\u001B[0m')
images = limageTags.split(',')
for image in images:
eachImage = image.split(':')
......@@ -84,15 +72,17 @@ class ClusterDeploy:
if imageName == 'mysql':
continue
# Check if image is exist on the Red Hat server, before pushing it to OC cluster
mySSH.command2("sudo podman image inspect --format='Size = {{.Size}} bytes' " + imageName + ":" + imageTag + f' | tee -a {lSourcePath}/archives/{imageName}_image_info.log', 60, silent=True)
if mySSH.cmd2Results.count('no such image') != 0:
subprocess.run(f'echo "IMAGENAME_TAG: {imageName}:{imageTag} > archives/{imageName}_image_info.log', shell=True)
res = subprocess.check_output("sudo podman image inspect --format='Size = {{.Size}} bytes' " + f'{imageName}:{imageTag} | tee -a archives/{imageName}_image_info.log', shell=True, universal_newlines=True)
subprocess.run(f"sudo podman image inspect --format='Date = {{.Created}}' " + f'{imageName}:{imageTag} | tee -a archives/{imageName}_image_info.log', shell=True)
res2 = re.search('no such image', str(res.strip()))
if res2 is not None:
logging.error(f'\u001B[1m No such image {imageName}]\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.cmd2Results)
if result is not None:
imageSize = float(result.group('size'))
res2 = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', str(res.strip()))
if res2 is not None:
imageSize = float(res2.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug(f'\u001B[1m {imageName} size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
......@@ -107,26 +97,27 @@ class ClusterDeploy:
logging.debug(f'{imageName} size is unknown')
# logging to OC Cluster and then switch to corresponding project
mySSH.command2(f'oc login -u {ocUserName} -p {ocPassword}', 6, silent=True)
if mySSH.cmd2Results.count('Login successful.') == 0:
res = subprocess.check_output(f'oc login -u {ocUserName} -p {ocPassword}', shell=True, universal_newlines=True)
res2 = re.search('Login successful.', str(res.strip()))
if res2 is None:
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug('\u001B[1m Login to OC Cluster Successfully\u001B[0m')
mySSH.command2(f'oc project {ocProjectName}', 6, silent=True)
if mySSH.cmd2Results.count(f'Already on project "{ocProjectName}"') == 0 and mySSH.cmd2Results.count(f'Now using project "{self.OCProjectName}"') == 0:
res = subprocess.check_output(f'oc project {ocProjectName}', shell=True, universal_newlines=True)
res2 = re.search(f'Already on project "{ocProjectName}"', str(res.strip()))
res3 = re.search(f'Now using project "{self.OCProjectName}"', str(res.strip()))
if res2 is None and res3 is None:
logging.error(f'\u001B[1m Unable to access OC project {ocProjectName}\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug(f'\u001B[1m Now using project {ocProjectName}\u001B[0m')
# Tag the image and push to the OC cluster
mySSH.command2('oc whoami -t | sudo podman login -u ' + ocUserName + ' --password-stdin https://default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/ --tls-verify=false', 6, silent=True)
if mySSH.cmd2Results.count('Login Succeeded!') == 0:
res = subprocess.check_output('oc whoami -t | sudo podman login -u ' + ocUserName + ' --password-stdin https://default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/ --tls-verify=false', shell=True, universal_newlines=True)
res2 = re.search('Login Succeeded!', str(res.strip()))
if res2 is None:
logging.error('\u001B[1m Podman Login to OC Cluster Registry Failed\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug('\u001B[1m Podman Login to OC Cluster Registry Successfully\u001B[0m')
......@@ -136,24 +127,26 @@ class ClusterDeploy:
imageTag = eachImage[1]
if imageName == 'mysql':
continue
mySSH.command2(f'cd {lSourcePath} && oc create -f openshift/{imageName}-image-stream.yml 2>&1 | tee -a archives/5gcn_imagestream_summary.txt', 6, silent=True)
if mySSH.cmd2Results.count('(AlreadyExists):') == 0 and mySSH.cmd2Results.count('created') == 0:
res = subprocess.check_output(f'oc create -f openshift/{imageName}-image-stream.yml 2>&1 | tee -a archives/5gcn_imagestream_summary.txt || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search('already exists', str(res.strip()))
res3 = re.search('created', str(res.strip()))
if res2 is None and res3 is None:
logging.error(f'\u001B[1m Image Stream "{imageName}" Creation Failed on OC Cluster {ocProjectName}\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug(f'\u001B[1m Image Stream "{imageName}" created on OC project {ocProjectName}\u001B[0m')
mySSH.command2(f'sudo podman tag {imageName}:{imageTag} default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag}', 6, silent=True)
mySSH.command2(f'sudo podman push default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag} --tls-verify=false 2>&1 | tee -a archives/5gcn_imagepush_summary.txt', 60, silent=True)
subprocess.run(f'sudo podman tag {imageName}:{imageTag} default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag}', shell=True)
res = subprocess.check_output(f'sudo podman push default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag} --tls-verify=false 2>&1 | tee -a archives/5gcn_imagepush_summary.txt', shell=True, universal_newlines=True)
time.sleep(10)
if mySSH.cmd2Results.count('Storing signatures') == 0:
res2 = re.search('Storing signatures', str(res.strip()))
if res2 is None:
logging.error(f'\u001B[1m Image "{imageName}" push to OC Cluster Registry Failed\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug(f'\u001B[1m Image "{imageName}" push to OC Cluster Registry Successfully\u001B[0m')
# Using helm charts deployment
logging.debug(f'\u001B[1m Deploying 5GCN Components on the Cluster using Helm Charts\u001B[0m')
passPods = 0
time.sleep(5)
for image in images:
......@@ -161,8 +154,8 @@ class ClusterDeploy:
imageName = eachImage[0]
imageTag = eachImage[1]
nameSufix = ''
mySSH.command(f'sed -i -e "s#PROJECT#{ocProjectName}#g" ./charts/{imageName}/values.yaml', '\$', 6, silent=True)
mySSH.command(f'sed -i -e "s#TAG#{imageTag}#g" ./charts/{imageName}/values.yaml', '\$', 6, silent=True)
subprocess.run(f'sed -i -e "s#PROJECT#{ocProjectName}#g" ./charts/{imageName}/values.yaml', shell=True)
subprocess.run(f'sed -i -e "s#TAG#{imageTag}#g" ./charts/{imageName}/values.yaml', shell=True)
if imageName == 'oai-nrf':
nameSufix = 'nrf'
elif imageName == 'oai-amf':
......@@ -171,64 +164,67 @@ class ClusterDeploy:
nameSufix = 'smf'
elif imageName == 'oai-spgwu-tiny':
nameSufix = 'spgwu'
mySSH.command2(f'cd {lSourcePath} && helm install {imageName} ./charts/{imageName}/ | tee -a archives/5gcn_helm_summary.txt 2>&1', 10, silent=True)
if mySSH.cmd2Results.count('STATUS: deployed') == 0:
res = subprocess.check_output(f'helm install {imageName} ./charts/{imageName}/ | tee -a archives/5gcn_helm_summary.txt 2>&1', shell=True, universal_newlines=True)
res2 = re.search('STATUS: deployed', str(res.strip()))
if res2 is None:
subprocess.run(f'echo "{imageName}: HELM KO" > archives/5gcn_helm_summary.txt', shell=True)
logging.error(f'\u001B[1m Deploying "{imageName}" Failed using helm chart on OC Cluster\u001B[0m')
logging.error(f'\u001B[1m 5GCN Deployment: KO \u001B[0m')
subprocess.run(f'echo "DEPLOYMENT: KO" > archives/deployment_status.log', shell=True)
self.UnDeploy_5gcn()
self.AnalyzeLogFile_5gcn()
sys.exit(-1)
else:
subprocess.run(f'echo "{imageName}: HELM OK" > archives/5gcn_helm_summary.txt', shell=True)
logging.debug(f'\u001B[1m Deployed "{imageName}" Successfully using helm chart\u001B[0m')
time.sleep(20)
mySSH.command2(f'oc get pods -o wide -l app.kubernetes.io/instance={imageName} | tee -a {lSourcePath}/archives/5gcn_pods_summary.txt', 6, silent=True)
result = re.search(f'{imageName}[\S\d\w]+', mySSH.cmd2Results)
podName = result.group(0)
res = subprocess.check_output(f'oc get pods -o wide -l app.kubernetes.io/instance={imageName} | tee -a archives/5gcn_pods_summary.txt', shell=True, universal_newlines=True)
res2 = re.search(f'{imageName}[\S\d\w]+', str(res.strip()))
podName = res2.group(0)
isRunning = False
count = 0
while count < 3 and isRunning == False:
time.sleep(10)
while count < 6 and isRunning == False:
time.sleep(5)
if imageName == 'mysql':
mySSH.command2(f'oc exec {podName} -i -t -- mysqladmin -u root --password=linux ping', 6, silent=True)
res = subprocess.check_output(f'oc exec {podName} -i -t -- mysqladmin -u root --password=linux ping || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
else:
mySSH.command2(f'oc exec {podName} -c {nameSufix} -it -- ps aux', 6, silent=True)
if mySSH.cmd2Results.count(f'oai_{nameSufix}') != 0 or mySSH.cmd2Results.count(f'mysqld is alive') != 0 :
res = subprocess.check_output(f'oc exec {podName} -c {nameSufix} -- ps aux || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'oai_{nameSufix}', str(res.strip()))
res3 = re.search(f'mysqld is alive', str(res.strip()))
if res2 is not None or res3 is not None:
logging.debug(f'\u001B[1m POD "{imageName}" Service Running Sucessfully\u001B[0m')
isRunning = True
passPods += 1
count +=1
if isRunning == False:
logging.error(f'\u001B[1m POD "{imageName}" Service Running FAILED \u001B[0m')
self.UnDeploy_5gcn()
self.AnalyzeLogFile_5gcn()
sys.exit(-1)
#self.UnDeploy_5gcn()
#sys.exit(-1)
if passPods == 5:
logging.debug(f'\u001B[1m 5gcn Deployment: OK \u001B[0m')
mySSH.command2(f'echo "DEPLOYMENT: OK" > {lSourcePath}/archives/deployment_status.log', 5, silent=True)
logging.debug(f'\u001B[1m 5GCN Deployment: OK \u001B[0m')
subprocess.run(f'echo "DEPLOYMENT: OK" > archives/deployment_status.log', shell=True)
else:
logging.error(f'\u001B[1m 5gcn Deployment: KO \u001B[0m')
mySSH.command2(f'echo "DEPLOYMENT: KO" > {lSourcePath}/archives/deployment_status.log', 5, silent=True)
logging.error(f'\u001B[1m 5GCN Deployment: KO \u001B[0m')
subprocess.run(f'echo "DEPLOYMENT: KO" > archives/deployment_status.log', shell=True)
self.UnDeploy_5gcn()
self.AnalyzeLogFile_5gcn()
sys.exit(-1)
subprocess.run('oc logout', shell=True)
def UnDeploy_5gcn(self):
mySSH = SSH.SSHConnection()
mySSH.open(self.remoteIPAdd, self.remoteUserName, self.remotePassword)
mySSH.command('cd ' + self.sourceCodePath, '\$', 5)
logging.debug('\u001B[1m UnDeploying the 5gcn\u001B[0m')
logging.debug('\u001B[1m UnDeploying the 5GCN\u001B[0m')
# logging to OC Cluster and then switch to corresponding project
mySSH.command2(f'oc login -u {self.OCUserName} -p {self.OCPassword}', 6, silent=True)
if mySSH.cmd2Results.count('Login successful.') == 0:
res = subprocess.check_output(f'oc login -u {self.OCUserName} -p {self.OCPassword}|| true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'Login successful.', str(res.strip()))
if res2 is None:
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug('\u001B[1m Login to OC Cluster Successfully\u001B[0m')
mySSH.command2(f'oc project {self.OCProjectName}', 6, silent=True)
if mySSH.cmd2Results.count(f'Already on project "{self.OCProjectName}"') == 0 and mySSH.cmd2Results.count(f'Now using project "{self.OCProjectName}"') == 0:
res = subprocess.check_output(f'oc project {self.OCProjectName} || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'Already on project "{self.OCProjectName}"', str(res.strip()))
res3 = re.search(f'Now using project "{self.OCProjectName}"', str(res.strip()))
if res2 is None and res3 is None:
logging.error(f'\u001B[1m Unable to access OC project {self.OCProjectName}\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug(f'\u001B[1m Now using project {self.OCProjectName}\u001B[0m')
......@@ -239,52 +235,46 @@ class ClusterDeploy:
eachImage = image.split(':')
imageName = eachImage[0]
imageTag = eachImage[1]
mySSH.command2(f'cd {self.sourceCodePath} && helm uninstall {imageName} | tee -a archives/5gcn_helm_summary.txt 2>&1', 6, silent=True)
if mySSH.cmd2Results.count(f'release "{imageName}" uninstalled') != 0:
res = subprocess.check_output(f'helm uninstall {imageName} | tee -a archives/5gcn_helm_summary.txt 2>&1', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'release "{imageName}" uninstalled', str(res.strip()))
res3 = re.search(f'Error: uninstall: Release not loaded: {imageName}: release: not found', str(res.strip()))
if res2 is not None:
logging.debug(f'\u001B[1m UnDeployed "{imageName}" Successfully on OC Cluster\u001B[0m')
elif mySSH.cmd2Results.count(f'Error: uninstall: Release not loaded: {imageName}: release: not found') != 0:
logging.debug(f'\u001B[1m No Resource "{imageName}" on OC Cluster\u001B[0m')
elif res3 is not None:
logging.debug(f'\u001B[1m No Resource "{imageName}" found on OC Cluster on project "{self.OCProjectName}"\u001B[0m')
else:
logging.error(f'\u001B[1m UnDeploying "{imageName}" Failed using helm chart on OC Cluster\u001B[0m')
time.sleep(2)
# Delete images and imagestream
if imageName == 'mysql':
continue
mySSH.command2(f'sudo podman rmi default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag}', 6, silent=True)
mySSH.command2(f'oc delete is {imageName}', 6, silent=True)
logging.debug(f'\u001B[1m Deleted the "{imageName}" Image and ImageStream\u001B[0m')
mySSH.command2('oc logout', 6, silent=True)
mySSH.close()
self.AnalyzeLogFile_5gcn()
res = subprocess.check_output(f'sudo podman rmi default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCProjectName}/{imageName}:{imageTag} || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search('in local storage: no such image', str(res.strip()))
res = subprocess.check_output(f'oc delete is {imageName} || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res3 = re.search('Error from server (NotFound):', str(res.strip()))
if res2 is None and res3 is None:
logging.debug(f'\u001B[1m Deleted the "{imageName}" Image and ImageStream\u001B[0m')
else:
logging.debug(f'\u001B[1m No Resource "{imageName}" Image and ImageStream found\u001B[0m')
subprocess.run('oc logout', shell=True)
def GetLogsConfigs(self):
lIpAddr = self.remoteIPAdd
lUserName = self.remoteUserName
lPassWord = self.remotePassword
lSourcePath = self.sourceCodePath
# copying the pcaps
lssh = subprocess.Popen(f'ls /nfs/prov-vols | grep {self.OCProjectName}-cn5g-pvc',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE, encoding='utf-8').stdout.readlines()
pStorage = str(lssh[0]).strip()
subprocess.Popen(f'mkdir -p archives/pcap',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
subprocess.Popen(f'cp /nfs/prov-vols/{pStorage}/* archives/pcap/',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
# copy the pods logs
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
mySSH.command2(f'oc login -u {self.OCUserName} -p {self.OCPassword}', 6, silent=True)
if mySSH.cmd2Results.count('Login successful.') == 0:
def GetLogsConfigsPcaps(self):
# copying the pod's logs, configs, pcaps
logging.debug(f'\u001B[1m Copying the PODs log, config & pcap\u001B[0m')
res = subprocess.check_output(f'oc login -u {self.OCUserName} -p {self.OCPassword}|| true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'Login successful.', str(res.strip()))
if res2 is None:
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug('\u001B[1m Login to OC Cluster Successfully\u001B[0m')
mySSH.command2(f'oc project {self.OCProjectName}', 6, silent=True)
if mySSH.cmd2Results.count(f'Already on project "{self.OCProjectName}"') == 0 and mySSH.cmd2Results.count(f'Now using project "{self.OCProjectName}"') == 0:
res = subprocess.check_output(f'oc project {self.OCProjectName} || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'Already on project "{self.OCProjectName}"', str(res.strip()))
res3 = re.search(f'Now using project "{self.OCProjectName}"', str(res.strip()))
if res2 is None and res3 is None:
logging.error(f'\u001B[1m Unable to access OC project {self.OCProjectName}\u001B[0m')
mySSH.close()
sys.exit(-1)
else:
logging.debug(f'\u001B[1m Now using project {self.OCProjectName}\u001B[0m')
mySSH.command2(f'mkdir -p {lSourcePath}/archives/logs', 6, silent=True)
subprocess.run(f'mkdir -p archives/pcap', shell=True)
subprocess.run(f'mkdir -p archives/logs', shell=True)
subprocess.run(f'mkdir -p archives/config', shell=True)
images = self.imageTags.split(',')
for image in images:
eachImage = image.split(':')
......@@ -301,30 +291,17 @@ class ClusterDeploy:
elif imageName == 'oai-spgwu-tiny':
nameSufix = 'spgwu'
nameSufix1 = 'spgwu-tiny'
mySSH.command2(f'oc get pods -o wide -l app.kubernetes.io/instance={imageName}', 6, silent=True)
result = re.search(f'{imageName}[\S\d\w]+', mySSH.cmd2Results)
podName = result.group(0)
mySSH.command2(f'oc logs {podName} {nameSufix} > {lSourcePath}/archives/logs/{imageName}_pod.log', 10, silent=True)
res = subprocess.check_output(f'oc get pods -o wide -l app.kubernetes.io/instance={imageName} || true', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
res2 = re.search(f'{imageName}[\S\d\w]+', str(res.strip()))
podName = res2.group(0)
subprocess.run(f'oc logs {podName} {nameSufix} > archives/logs/{imageName}_pod.log', shell=True)
time.sleep(1)
if imageName == 'oai-spgwu-tiny':
mySSH.command2(f'oc cp {podName}:/openair-{nameSufix1}/etc/ {lSourcePath}/archives/config/ -c {nameSufix}', 6, silent=True)
subprocess.run(f'oc cp {podName}:/openair-{nameSufix1}/etc/ archives/config/ -c {nameSufix}', shell=True)
subprocess.run(f'oc cp {podName}:/pcap/ archives/pcap/ -c tcpdump', shell=True)
else:
mySSH.command2(f'oc cp {podName}:/openair-{nameSufix}/etc/ {lSourcePath}/archives/config/ -c {nameSufix}', 6, silent=True)
mySSH.command('oc logout', '\$', 6)
mySSH.close()
def AnalyzeLogFile_5gcn(self):
lIpAddr = self.remoteIPAdd
lUserName = self.remoteUserName
lPassWord = self.remotePassword
lSourcePath = self.sourceCodePath
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
mySSH.command('cd ' + lSourcePath, '\$', 5)
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/archives/*', '../archives/')
mySSH.command('rm -rf ./archives', '\$', 5)
mySSH.command('oc logout', '\$', 6)
mySSH.close()
subprocess.run(f'oc cp {podName}:/openair-{nameSufix}/etc/ archives/config/ -c {nameSufix}', shell=True)
subprocess.run('oc logout', shell=True)
def Usage():
print('----------------------------------------------------------------------------------------------------------------------')
......@@ -335,9 +312,6 @@ def Usage():
print(' --help Show this help.')
print('---------------------------------------------------------------------------------------------- Mandatory Options -----')
print(' --mode=[Deploy/UnDeploy]')
print(' --remoteIPAdd=[IP Address of remote Server]')
print(' --remoteUserName=[UserName of remote Server]')
print(' --remotePassword=[Password of remote Server]')
print(' --OCUserName=[Cluster UserName]')
print(' --OCPassword=[Cluster Password]')
print(' --OCProjectName=[Cluster Project name]')
......@@ -345,6 +319,7 @@ def Usage():
print('------------------------------------------------------------------------------------------------- Actions Syntax -----')
print('python3 helmDeploy.py --mode=Deploy [Mandatory Options]')
print('python3 helmDeploy.py --mode=UnDeploy [Mandatory Options]')
print('python3 helmDeploy.py --mode=GetLogs [Mandatory Options]')
......@@ -366,15 +341,6 @@ while len(argvs) > 1:
elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
CN.mode = matchReg.group(1)
elif re.match('^\-\-remoteIPAdd=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-remoteIPAdd=(.+)$', myArgv, re.IGNORECASE)
CN.remoteIPAdd = matchReg.group(1)
elif re.match('^\-\-remoteUserName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-remoteUserName=(.+)$', myArgv, re.IGNORECASE)
CN.remoteUserName = matchReg.group(1)
elif re.match('^\-\-remotePassword=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-remotePassword=(.+)$', myArgv, re.IGNORECASE)
CN.remotePassword = matchReg.group(1)
elif re.match('^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match('^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE)
CN.OCUserName = matchReg.group(1)
......@@ -395,4 +361,4 @@ if CN.mode == 'Deploy':
elif CN.mode == 'UnDeploy':
CN.UnDeploy_5gcn()
elif CN.mode == 'GetLogs':
CN.GetLogsConfigs()
\ No newline at end of file
CN.GetLogsConfigsPcaps()
\ No newline at end of file
ci-scripts/sshconnection.py deleted 100644 → 0
+ 0
− 259
View file @ 6473621a
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
# Required Python Version
# Python 3.x
#
# Required Python Package
# pexpect
#---------------------------------------------------------------------
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import pexpect # pexpect
import logging
import time # sleep
import re
import subprocess
import sys
#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
def __init__(self):
self.ssh = ''
self.picocom_closure = False
self.ipaddress = ''
self.username = ''
self.cmd2Results = ''
def disablePicocomClosure(self):
self.picocom_closure = False
def enablePicocomClosure(self):
self.picocom_closure = True
def open(self, ipaddress, username, password):
count = 0
connect_status = False
while count < 4:
self.ssh = pexpect.spawn('ssh -o PubkeyAuthentication=no {}@{}'.format(username,ipaddress))
self.ssh.timeout = 5
self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
if self.sshresponse == 0:
self.ssh.sendline('yes')
self.sshresponse = self.ssh.expect(['password:', username + '@'])
if self.sshresponse == 0:
self.ssh.sendline(password)
self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if self.sshresponse == 0:
count = 10
connect_status = True
else:
logging.debug('self.sshresponse = ' + str(self.sshresponse))
elif self.sshresponse == 1:
self.ssh.sendline(password)
self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if self.sshresponse == 0:
count = 10
connect_status = True
else:
logging.debug('self.sshresponse = ' + str(self.sshresponse))
elif self.sshresponse == 2:
# Checking if we are really on the remote client defined by its IP address
self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:|inet "', '\$', 5)
result = re.search(str(ipaddress), str(self.ssh.before))
if result is None:
self.close()
else:
count = 10
connect_status = True
else:
# debug output
logging.debug(str(self.ssh.before))
logging.debug('self.sshresponse = ' + str(self.sshresponse))
# adding a tempo when failure
if not connect_status:
time.sleep(1)
count += 1
if connect_status:
pass
else:
sys.exit('SSH Connection Failed')
self.ipaddress = ipaddress
self.username = username
def cde_check_value(self, commandline, expected, timeout):
logging.debug(commandline)
self.ssh.timeout = timeout
self.ssh.sendline(commandline)
expected.append(pexpect.EOF)
expected.append(pexpect.TIMEOUT)
self.sshresponse = self.ssh.expect(expected)
return self.sshresponse
def command(self, commandline, expectedline, timeout, silent=False, resync=False):
if not silent:
logging.debug(commandline)
self.ssh.timeout = timeout
# Nasty patch when pexpect output is out of sync.
# Much pronounced when running back-to-back-back oc commands
if resync:
self.ssh.send(commandline)
self.ssh.expect([commandline, pexpect.TIMEOUT])
self.ssh.send('\r\n')
self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
else:
self.ssh.sendline(commandline)
self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
if self.sshresponse == 0:
return 0
elif self.sshresponse == 1:
logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
logging.debug('Expected Line : ' + expectedline)
logging.debug(str(self.ssh.before))
sys.exit(self.sshresponse)
elif self.sshresponse == 2:
logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
logging.debug('Expected Line : ' + expectedline)
result = re.search('ping |iperf |picocom', str(commandline))
if result is None:
logging.debug(str(self.ssh.before))
sys.exit(self.sshresponse)
else:
return -1
else:
logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
logging.debug('Expected Line : ' + expectedline)
sys.exit(self.sshresponse)
def command2(self, commandline, timeout, silent=False):
if not silent:
logging.debug(commandline)
self.cmd2Results = ''
myHost = self.username + '@' + self.ipaddress
# CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS
# BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST
# OTHERWISE IT WON'T WORK
lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
self.cmd2Results = str(lSsh.stdout.readlines())
def close(self):
self.ssh.timeout = 5
self.ssh.sendline('exit')
self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
self.ipaddress = ''
self.username = ''
if self.sshresponse == 0:
pass
elif self.sshresponse == 1:
if not self.picocom_closure:
logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m')
else:
logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m')
def copyin(self, ipaddress, username, password, source, destination):
count = 0
copy_status = False
logging.debug('scp -r '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
while count < 10:
scp_spawn = pexpect.spawn('scp -r '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 100)
scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0:
scp_spawn.sendline('yes')
scp_spawn.expect('password:')
scp_spawn.sendline(password)
scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0:
count = 10
copy_status = True
else:
logging.debug('1 - scp_response = ' + str(scp_response))
elif scp_response == 1:
scp_spawn.sendline(password)
scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0 or scp_response == 3:
count = 10
copy_status = True
else:
logging.debug('2 - scp_response = ' + str(scp_response))
elif scp_response == 2:
count = 10
copy_status = True
else:
logging.debug('3 - scp_response = ' + str(scp_response))
# adding a tempo when failure
if not copy_status:
time.sleep(1)
count += 1
if copy_status:
return 0
else:
return -1
def copyout(self, ipaddress, username, password, source, destination):
count = 0
copy_status = False
logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
while count < 4:
scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 100)
scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0:
scp_spawn.sendline('yes')
scp_spawn.expect('password:')
scp_spawn.sendline(password)
scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0:
count = 10
copy_status = True
else:
logging.debug('1 - scp_response = ' + str(scp_response))
elif scp_response == 1:
scp_spawn.sendline(password)
scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
if scp_response == 0 or scp_response == 3:
count = 10
copy_status = True
else:
logging.debug('2 - scp_response = ' + str(scp_response))
elif scp_response == 2:
count = 10
copy_status = True
else:
logging.debug('3 - scp_response = ' + str(scp_response))
# adding a tempo when failure
if not copy_status:
time.sleep(1)
count += 1
if copy_status:
pass
else:
sys.exit('SCP failed')
def getBefore(self):
return str(self.ssh.before)
ci-scripts/toCheckDSTesterResult1.py
+ 5
− 5
View file @ a75f2a82
......@@ -24,6 +24,7 @@ import re
import sys
import subprocess
import yaml
import logging
locexist = False
try:
......@@ -33,17 +34,16 @@ try:
result = re.search('(?:\/.+?\/)(.+?)(?:\/.+)', str(line))
if result:
result1 = re.search('^(.*/)([^/]*)$', str(result.group(0)))
print(result1.group(1))
subprocess.Popen(f'cp {result1.group(1)}* DS-TEST-RESULTS/',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
locexist = True
except IOError:
print("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt")
sys.exit("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt")
if locexist:
try:
with open('DS-TEST-RESULTS/5gcn.yaml') as f:
data = yaml.full_load(f)
data = yaml.load(f)
if data["final-result"] == 'fail':
sys.exit(-1)
sys.exit('DsTester final result FAILED')
except IOError:
print("File not accessible to check DSTester result: DS-TEST-RESULTS/5gcn.yaml")
sys.exit("File not accessible to check DSTester result: DS-TEST-RESULTS/5gcn.yaml")
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment