Commit 6aa12648 authored by Raphael Defosseux's avatar Raphael Defosseux
Browse files

[CI][Docker] Building docker images in python framework


- only on Ubuntu and Docker
- No back ground computing to paralellize RHEL/Ubuntu
- No image clean up
Signed-off-by: Raphael Defosseux's avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent df334999
......@@ -98,8 +98,9 @@ def AssignParams(params_dict):
def GetParametersFromXML(action):
if action == 'Build_eNB':
if action == 'Build_eNB' or action == 'Build_Image':
RAN.Build_eNB_args=test.findtext('Build_eNB_args')
RAN.imageKind=test.findtext('kind')
forced_workspace_cleanup = test.findtext('forced_workspace_cleanup')
if (forced_workspace_cleanup is None):
RAN.Build_eNB_forced_workspace_cleanup=False
......@@ -127,8 +128,9 @@ def GetParametersFromXML(action):
else:
RAN.backgroundBuild=False
elif action == 'WaitEndBuild_eNB':
elif action == 'WaitEndBuild_eNB' or action == 'WaitEndBuild_Image':
RAN.Build_eNB_args=test.findtext('Build_eNB_args')
RAN.imageKind=test.findtext('kind')
eNB_instance=test.findtext('eNB_instance')
if (eNB_instance is None):
RAN.eNB_instance=0
......@@ -640,8 +642,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
break
if action == 'Build_eNB':
RAN.BuildeNB()
elif action == 'Build_Image':
RAN.BuildImage()
elif action == 'WaitEndBuild_eNB':
RAN.WaitBuildeNBisFinished()
elif action == 'WaitEndBuild_Image':
RAN.WaitBuildImageisFinished()
elif action == 'Initialize_eNB':
check_eNB = False
check_OAI_UE = False
......
......@@ -77,6 +77,7 @@ class RANManagement():
self.backgroundBuildTestId = ['', '', '']
self.Build_eNB_forced_workspace_cleanup = False
self.Initialize_eNB_args = ''
self.imageKind = ''
self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
self.eNB_instance = 0
self.eNB_serverId = ['', '', '']
......@@ -318,6 +319,171 @@ class RANManagement():
self.htmlObj.CreateHtmlTabFooter(False)
sys.exit(1)
def BuildImage(self):
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
if self.eNB_serverId[self.eNB_instance] == '0':
lIpAddr = self.eNBIPAddress
lUserName = self.eNBUserName
lPassWord = self.eNBPassword
lSourcePath = self.eNBSourceCodePath
elif self.eNB_serverId[self.eNB_instance] == '1':
lIpAddr = self.eNB1IPAddress
lUserName = self.eNB1UserName
lPassWord = self.eNB1Password
lSourcePath = self.eNB1SourceCodePath
elif self.eNB_serverId[self.eNB_instance] == '2':
lIpAddr = self.eNB2IPAddress
lUserName = self.eNB2UserName
lPassWord = self.eNB2Password
lSourcePath = self.eNB2SourceCodePath
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
logging.debug('Building on server: ' + lIpAddr)
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
imageNames = []
result = re.search('eNB', self.imageKind)
if result is not None:
imageNames.append(('oai-enb', 'eNB'))
else:
result = re.search('gNB', self.imageKind)
if result is not None:
imageNames.append(('oai-gnb', 'gNB'))
else:
result = re.search('all', self.imageKind)
if result is not None:
imageNames.append(('oai-enb', 'eNB'))
imageNames.append(('oai-gnb', 'gNB'))
if len(imageNames) == 0:
imageNames.append(('oai-enb', 'eNB'))
# Workaround for some servers, we need to erase completely the workspace
if self.Build_eNB_forced_workspace_cleanup:
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
if self.htmlObj is not None:
self.testCase_id = self.htmlObj.testCase_id
else:
self.testCase_id = '000000'
# on RedHat/CentOS .git extension is mandatory
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
if result is not None:
full_ran_repo_name = self.ranRepository
else:
full_ran_repo_name = self.ranRepository + '.git'
mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
mySSH.command('cd ' + lSourcePath, '\$', 5)
mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
# Raphael: here add a check if git clone or git fetch went smoothly
mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
# if the commit ID is provided use it to point to it
if self.ranCommitID != '':
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
# if the branch is not develop, then it is a merge request and we need to do
# the potential merge. Note that merge conflicts should already been checked earlier
imageTag = 'develop'
if (self.ranAllowMerge):
imageTag = 'ci-temp'
if self.ranTargetBranch == '':
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
else:
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
# Let's remove any previous run artifacts if still there
mySSH.command('docker image prune --force', '\$', 5)
mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5)
for image,pattern in imageNames:
mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5)
# Build the shared image
mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800)
# Build the target image(s)
previousImage='ran-build:' + imageTag
danglingShaOnes=[]
for image,pattern in imageNames:
mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600)
# Retrieving the dangling image(s) for the log collection
mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5)
result = re.search('sha=(?P<imageShaOne>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
if result is not None:
danglingShaOnes.append((image, result.group('imageShaOne')))
previousImage=image + ':' + imageTag
imageTag = 'ci-temp'
# First verify if images were properly created.
status = True
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5)
if mySSH.getBefore().count('No such object') != 0:
logging.error('Could not build properly ran-build')
status = False
else:
result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
if result is not None:
imageSize = float(result.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
else:
logging.debug('ran-build size is unknown')
for image,pattern in imageNames:
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
if mySSH.getBefore().count('No such object') != 0:
logging.error('Could not build properly ' + image)
status = False
else:
result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
if result is not None:
imageSize = float(result.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
else:
logging.debug('ran-build size is unknown')
if not status:
mySSH.close()
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
if self.htmlObj is not None:
self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
self.htmlObj.CreateHtmlTabFooter(False)
sys.exit(1)
# Recover build logs, for the moment only possible when build is successful
mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5)
mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5)
mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/ran-build', '\$', 5)
mySSH.command('docker rm -f test', '\$', 5)
for image,shaone in danglingShaOnes:
mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
mySSH.command('docker create --name test ' + shaone, '\$', 5)
mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/' + image, '\$', 5)
mySSH.command('docker rm -f test', '\$', 5)
mySSH.command('docker image prune --force', '\$', 5)
mySSH.close()
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
if self.htmlObj is not None:
self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
def InitializeeNB(self):
if self.eNB_serverId[self.eNB_instance] == '0':
lIpAddr = self.eNBIPAddress
......
- Build_PhySim
- Run_PhySim
- Build_eNB
- Build_Image
- WaitEndBuild_eNB
- WaitEndBuild_Image
- Initialize_eNB
- Terminate_eNB
- Initialize_UE
......
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
000004
-->
<testCaseList>
<htmlTabRef>build-tab</htmlTabRef>
<htmlTabName>Build</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
000001
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="000001">
<class>Build_Image</class>
<desc>Build eNB Image</desc>
<kind>all</kind>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<backgroundBuild>True</backgroundBuild>
</testCase>
<testCase id="000004">
<class>WaitEndBuild_Image</class>
<desc>Wait for end of Build eNB</desc>
<kind>eNB</kind>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
</testCaseList>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment