Commit 713cd711 authored by Raphael Defosseux's avatar Raphael Defosseux
Browse files

feat(ci): adding sanity check deployment, no proper verification


Signed-off-by: Raphael Defosseux's avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent 1f117b51
......@@ -236,17 +236,21 @@ pipeline {
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service vpp-upf --upf_tag ' + upf_tag
// SMF is dependant on AMF and EXT-DN: they will be deploy as well
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service oai-smf'
// GNB SIM as RAN emulator
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp'
// Check if deployment is OK
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all'
// Undeploy once OK
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp'
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all'
}
}
post {
unsuccessful {
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all'
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all || true'
sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all || true'
}
}
}
......
......@@ -656,10 +656,92 @@ class HtmlReport():
def testSummaryHeader(self):
self.file.write(' <h2>Test Summary</h2>\n')
self.file.write(' <div class="alert alert-warning">\n')
self.file.write(' <strong>Not performed yet. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
cwd = os.getcwd()
if os.path.isfile(cwd + '/archives/deployment_status.log'):
cmd = 'egrep -c "DEPLOYMENT: OK" archives/deployment_status.log || true'
status = False
ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8')
if ret.stdout is not None:
status = True
if status:
self.file.write(' <div class="alert alert-success">\n')
self.file.write(' <strong>Successful Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
self.file.write(' </div>\n')
else:
self.file.write(' <div class="alert alert-danger">\n')
self.file.write(' <strong>Failed Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
self.file.write(' </div>\n')
else:
self.file.write(' <div class="alert alert-warning">\n')
self.file.write(' <strong>Not performed. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n')
self.file.write(' </div>\n')
self.file.write(' <br>\n')
self.file.write(' <button data-toggle="collapse" data-target="#deployment-details">More details on Deployment</button>\n')
self.file.write(' <br>\n')
self.file.write(' <div id="deployment-details" class="collapse">\n')
self.file.write(' <br>\n')
self.file.write(' <table class="table-bordered" width = "80%" align = "center" border = 1>\n')
self.file.write(' <tr bgcolor = "#33CCFF" >\n')
self.file.write(' <th>Container Name</th>\n')
self.file.write(' <th>Used Image Tag</th>\n')
self.file.write(' <th>Image Creation Date</th>\n')
self.file.write(' <th>Used Image Size</th>\n')
self.file.write(' </tr>\n')
self.addImageRow('mysql')
self.addImageRow('oai_nrf')
self.addImageRow('oai_amf')
self.addImageRow('oai_smf')
self.addImageRow('oai_upf_vpp')
self.file.write(' </table>\n')
self.file.write(' </div>\n')
def addImageRow(self, imageInfoPrefix):
cwd = os.getcwd()
if imageInfoPrefix == 'oai_amf':
containerName = 'oai-amf'
tagPattern = 'OAI_AMF_TAG'
if imageInfoPrefix == 'oai_smf':
containerName = 'oai-smf'
tagPattern = 'OAI_SMF_TAG'
if imageInfoPrefix == 'oai_nrf':
containerName = 'oai-nrf'
tagPattern = 'OAI_NRF_TAG'
if imageInfoPrefix == 'oai_upf_vpp':
containerName = 'vpp-upf'
tagPattern = 'OAI_UPF_VPP_TAG'
if imageInfoPrefix == 'mysql':
containerName = imageInfoPrefix
tagPattern = 'MYSQL_TAG'
if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'):
usedTag = ''
createDate = ''
size = ''
with open(cwd + '/archives/' + imageInfoPrefix + '_image_info.log') as imageLog:
for line in imageLog:
line = line.strip()
result = re.search(tagPattern + ': (?P<tag>[a-zA-Z0-9\.\-\_:]+)', line)
if result is not None:
usedTag = result.group('tag')
result = re.search('Date = (?P<date>[a-zA-Z0-9\-\_:]+)', line)
if result is not None:
createDate = result.group('date')
result = re.search('Size = (?P<size>[0-9]+) bytes', line)
if result is not None:
sizeInt = int(result.group('size'))
if sizeInt < 1000000:
sizeInt = int(sizeInt / 1000)
size = str(sizeInt) + ' kB'
else:
sizeInt = int(sizeInt / 1000000)
size = str(sizeInt) + ' MB'
imageLog.close()
self.file.write(' <tr>\n')
self.file.write(' <td>' + containerName + '</td>\n')
self.file.write(' <td>' + usedTag + '</td>\n')
self.file.write(' <td>' + createDate + '</td>\n')
self.file.write(' <td>' + size + '</td>\n')
self.file.write(' </tr>\n')
def testSummaryFooter(self):
self.file.write(' <br>\n')
......
......@@ -23,8 +23,10 @@ import argparse
import logging
import os
import re
import sys
import shutil
import subprocess
import sys
import time
logging.basicConfig(
level=logging.DEBUG,
......@@ -36,12 +38,13 @@ def main() -> None:
args = _parse_args()
if args.mode == 'Deploy':
deployObject(args.service, args.upf_tag)
deployObject(args.service, args.upf_tag, args.docker_compose)
if args.mode == 'Check':
logging.info ('Performing Check')
checkDeployment(args.service, args.docker_compose)
if args.mode == 'UnDeploy':
logging.info ('Un-Deploying ' + args.service)
undeployObject(args.service, args.docker_compose)
def _parse_args() -> argparse.Namespace:
"""Parse the command line args
......@@ -82,10 +85,103 @@ def _parse_args() -> argparse.Namespace:
)
return parser.parse_args()
def deployObject(service, tag):
def deployObject(service, tag, compose_file):
logging.info ('Deploying ' + service)
if service == 'gnbsim-vpp':
logging.info ('Not available for the moment')
return
if not os.path.isdir('./archives'):
subprocess_run_w_echo('mkdir -p ./archives')
os.chdir('docker-compose')
if not os.path.exists('./ci-' + compose_file):
logging.debug('cp ./' + compose_file + ' ./ci-' + compose_file)
shutil.copyfile('./' + compose_file, './ci-' + compose_file)
expect_healthy = 0
nb_healthy = 0
cnt = 0
# mysql does not require image tag update
# Also starting a capture on the public docker network
if service == 'mysql':
subprocess_run_w_echo('echo "MYSQL_TAG: mysql:5.7" > ../archives/mysql_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" mysql:5.7 >> ../archives/mysql_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" mysql:5.7 >> ../archives/mysql_image_info.log')
subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
expect_healthy = 1
subprocess_run_w_echo('sudo nohup tshark -i cn5g-public -f "not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com" -w /tmp/ci-upf-vpp-sanity.pcap > /dev/null 2>&1 &')
# We might need tag manipulations
if service == 'oai-nrf':
subprocess_run_w_echo('echo "OAI_NRF_TAG: oai-nrf:develop" > ../archives/oai_nrf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-nrf:develop >> ../archives/oai_nrf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-nrf:develop >> ../archives/oai_nrf_image_info.log')
subprocess_run_w_echo('sed -i -e "s#oai-nrf:latest#oai-nrf:develop#" ./ci-' + compose_file)
subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
expect_healthy = 2
if service == 'vpp-upf':
subprocess_run_w_echo('echo "OAI_UPF_VPP_TAG: oai-upf-vpp:' + tag + '" > ../archives/oai_upf_vpp_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log')
subprocess_run_w_echo('sed -i -e "s#oai-upf-vpp:latest#oai-upf-vpp:' + tag + '#" ./ci-' + compose_file)
subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
expect_healthy = 3
if service == 'oai-smf':
subprocess_run_w_echo('echo "OAI_AMF_TAG: oai-amf:develop" > ../archives/oai_amf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-amf:develop >> ../archives/oai_amf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-amf:develop >> ../archives/oai_amf_image_info.log')
subprocess_run_w_echo('echo "OAI_SMF_TAG: oai-smf:develop" > ../archives/oai_smf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-smf:develop >> ../archives/oai_smf_image_info.log')
subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-smf:develop >> ../archives/oai_smf_image_info.log')
subprocess_run_w_echo('sed -i -e "s#oai-amf:latest#oai-amf:develop#" -e "s#oai-smf:latest#oai-smf:develop#" ./ci-' + compose_file)
subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service)
expect_healthy = 5
# Counting the number of healthy containers
cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a'
while (nb_healthy != expect_healthy) or (cnt > 50):
time.sleep(5)
cnt += 1
res = subprocess.check_output(cmd, shell=True, universal_newlines=True)
nb_healthy = 0
for line in res.split('\n'):
result = re.search('Up.*healthy', line)
if result is not None:
nb_healthy += 1
if nb_healthy == expect_healthy:
logging.info('OK')
elif cnt > 50:
logging.error('KO')
sys.exit(-1)
def undeployObject(service, compose_file):
logging.info ('Un-Deploying ' + service)
if service == 'gnbsim-vpp':
logging.info ('Not available for the moment')
return
os.chdir('docker-compose')
subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' down')
def checkDeployment(service, compose_file):
logging.info ('Performing Check')
subprocess_run_w_echo('sudo chmod 666 /tmp/ci-upf-vpp-sanity.pcap')
if not os.path.isdir('./archives'):
subprocess_run_w_echo('mkdir -p ./archives')
if os.path.exists('/tmp/ci-upf-vpp-sanity.pcap'):
subprocess_run_w_echo('cp /tmp/ci-upf-vpp-sanity.pcap archives')
os.chdir('docker-compose')
subprocess_run_w_echo ('ls')
cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a'
res = subprocess.check_output(cmd, shell=True, universal_newlines=True)
for line in res.split('\n'):
result = re.search('Up.*healthy', line)
if result is not None:
subprocess_run_w_echo('docker logs ' + line.split(' ')[0] + ' > ../archives/' + line.split(' ')[0] + '.log 2>&1')
subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: OK" > ../archives/deployment_status.log')
def subprocess_run_w_echo(cmd):
logging.debug(cmd)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment