Jenkinsfile-GitLab-Docker 18.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/bin/groovy
/*
 * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The OpenAirInterface Software Alliance licenses this file to You under
 * the OAI Public License, Version 1.1  (the "License"); you may not use this file
 * except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.openairinterface.org/?page_id=698
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *-------------------------------------------------------------------------------
 * For more information about the OpenAirInterface (OAI) Software Alliance:
 *      contact@openairinterface.org
 */

//-------------------------------------------------------------------------------
// Abstraction function to send social media messages:
// like on Slack or Mattermost
def sendSocialMediaMessage(pipeChannel, pipeColor, pipeMessage) {
  if (params.pipelineUsesSlack != null) {
    if (params.pipelineUsesSlack) {
      slackSend channel: pipeChannel, color: pipeColor, message: pipeMessage
    }
  }
}

// Location of the CN executor node
def cn_ci_host = params.Host_CN_CI_Server

// for lock
def cn_ci_resource = params.DockerContainers
def ds_tester_ci_resource = params.DsTester

41
42
43
// When triggered by upstream, specify which tag to use
def upstreamTagToUse = params.upstreamTagToUse

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
// Location of the 2nd CN executor
def new_host_flag = false
def new_host = ""
def new_host_user = ""

// Location of the CN tester
def dsT_host_flag = false
def dsT_host = ""
def dsT_host_user = ""
def dsT_host_ip_addr = ""

// Flags
def scmEvent = false
def upstreamEvent = false

// Default tags / branches  --> could be passed on by upstream job or by PR content
60
61
62
63
64
65
66
67
def nrfTag = params.nrfTag
def nrfBranch = params.nrfBranch
def amfTag = params.amfTag
def amfBranch = params.amfBranch
def smfTag = params.smfTag
def smfBranch = params.smfBranch
def spgwuTag = params.spgwuTag
def spgwuBranch = params.spgwuBranch
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//-------------------------------------------------------------------------------
// Pipeline start
pipeline {
  agent {
    label cn_ci_host
  }
  options {
    disableConcurrentBuilds()
    timestamps()
    ansiColor('xterm')
    lock(cn_ci_resource)
  }
  stages {
    stage ('Verify Parameters') {
      steps {
        script {
          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'

          JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
          JOB_TIMESTAMP = JOB_TIMESTAMP.trim()

          if (params.Host_CN_CI_2nd_Server_Flag != null) {
            new_host_flag = params.Host_CN_CI_2nd_Server_Flag
            if (new_host_flag) {
              new_host = params.Host_CN_CI_2nd_Server
              new_host_user = params.Host_CN_CI_2nd_Server_Login
              echo "1st Node   is ${NODE_NAME}"
              echo "2nd Node   is ${new_host}"
            } else {
              echo "Node       is ${NODE_NAME}"
            }
          } else {
            echo "Node       is ${NODE_NAME}"
          }
          if (params.DS_Tester_Server_Flag != null) {
104
            dsT_host_flag = params.DS_Tester_Server_Flag
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
            if (dsT_host_flag) {
              def allParametersPresent = true
              if (params.DS_Tester_Server_Name == null) {
                allParametersPresent = false
              } else {
                dsT_host = params.DS_Tester_Server_Name
              }
              if (params.DS_Tester_Server_Login == null) {
                allParametersPresent = false
              } else {
                dsT_host_user = params.DS_Tester_Server_Login
              }
              if (params.DS_Tester_Server_IP_Addr == null) {
                allParametersPresent = false
              } else {
                dsT_host_ip_addr = params.DS_Tester_Server_IP_Addr
              }
              if (allParametersPresent) {
                echo "DS Tester  is on ${dsT_host}"
              } else {
                echo "Some DS Tester parameters are missing!"
                sh "./ci-scripts/fail.sh"
              }
            }
          }

          // Find out the cause of the trigger
          for (cause in currentBuild.getBuildCauses()) {
            if (cause.toString() ==~ /.*UpstreamCause.*/) {
              upstreamEvent = true
Raphael Defosseux's avatar
Raphael Defosseux committed
135
136
            //} else {
            //  scmEvent = true
137
138
139
140
            }
          }

          if (upstreamEvent) {
141
142
143
144
145
146
147
148
            if (params.NRF_TAG != null) {
              nrfTag = params.NRF_TAG
              echo "Upstream Job passed NRF_TAG to use: ${nrfTag}"
            }
            if (params.NRF_BRANCH != null) {
              nrfBranch = params.NRF_BRANCH
              echo "Upstream Job passed NRF_BRANCH to use: ${nrfBranch}"
            }
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
            if (params.AMF_TAG != null) {
              amfTag = params.AMF_TAG
              echo "Upstream Job passed AMF_TAG to use: ${amfTag}"
            }
            if (params.AMF_BRANCH != null) {
              amfBranch = params.AMF_BRANCH
              echo "Upstream Job passed AMF_BRANCH to use: ${amfBranch}"
            }
            if (params.SMF_TAG != null) {
              smfTag = params.SMF_TAG
              echo "Upstream Job passed SMF_TAG to use: ${smfTag}"
            }
            if (params.SMF_BRANCH != null) {
              smfBranch = params.SMF_BRANCH
              echo "Upstream Job passed SMF_BRANCH to use: ${smfBranch}"
            }
            sh "git clean -x -d -f > /dev/null 2>&1"
166
167
            sh "git fetch --prune > /dev/null 2>&1"
            sh 'git checkout -f ' + upstreamTagToUse
Raphael Defosseux's avatar
Raphael Defosseux committed
168
            sh "zip -r -qq oai-cn5g-fed.zip .git"
169
            sh "mkdir -p archives DS-TEST-RESULTS"
170
            sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
171
172
            if (new_host_flag) {
              // Prepare the workspace in remote server
173
              copyTo2ndServer('oai-cn5g-fed.zip', new_host_flag, new_host_user, new_host)
174
175
              myShCmd('git clean -x -d -f > /dev/null 2>&1', new_host_flag, new_host_user, new_host)
              myShCmd('mkdir -p archives DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
176
              myShCmd('./scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch, new_host_flag, new_host_user, new_host)
177
178
            }
          }
179
180
          if (scmEvent) {
            sh "git clean -x -d -f > /dev/null 2>&1"
181
182
183
            if ("MERGE".equals(env.gitlabActionType)) {
              sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}"
            }
Raphael Defosseux's avatar
Raphael Defosseux committed
184
            sh "zip -r -qq oai-cn5g-fed.zip .git"
185
            sh "mkdir -p archives DS-TEST-RESULTS"
186
            sh './scripts/syncComponents.sh'
187
188
189
190
            if (new_host_flag) {
              // Prepare the workspace in remote server
              copyTo2ndServer('oai-cn5g-fed.zip', new_host_flag, new_host_user, new_host)
              myShCmd('git clean -x -d -f > /dev/null 2>&1', new_host_flag, new_host_user, new_host)
191
192
193
              if ("MERGE".equals(env.gitlabActionType)) {
                myShCmd("./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}", new_host_flag, new_host_user, new_host)
              }
194
              myShCmd('mkdir -p archives DS-TEST-RESULTS', new_host_flag, new_host_user, new_host)
195
              myShCmd('./scripts/syncComponents.sh', new_host_flag, new_host_user, new_host)
196
197
            }
          }
ismail's avatar
ismail committed
198
199
          if ((!upstreamEvent) && (!scmEvent)) {
            sh "git clean -x -d -f > /dev/null 2>&1"
200
            sh './scripts/syncComponents.sh --nrf-branch ' + nrfBranch + ' --amf-branch ' + amfBranch + ' --smf-branch ' + smfBranch + ' --spgwu-tiny-branch ' + spgwuBranch
ismail's avatar
ismail committed
201
202
            sh "mkdir -p archives DS-TEST-RESULTS"
          }
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
          // Verify that the images are available
          try {
            sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log'
            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
          } catch (Exception e) {
            error "OAI NRF Image tag to test does not exist!"
          }
          try {
            sh 'echo "OAI_AMF_TAG: oai-amf:' + amfTag +'" > archives/oai_amf_image_info.log'
            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-amf:' + amfTag + ' >> archives/oai_amf_image_info.log'
          } catch (Exception e) {
            error "OAI AMF Image tag to test does not exist!"
          }
          try {
            sh 'echo "OAI_SMF_TAG: oai-smf:' + smfTag +'" > archives/oai_smf_image_info.log'
            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-smf:' + smfTag + ' >> archives/oai_smf_image_info.log'
          } catch (Exception e) {
            error "OAI SMF Image tag to test does not exist!"
          }
          try {
226
227
228
            sh 'echo "OAI_SPGWU_TAG: oai-spgwu-tiny:' + spgwuTag +'" > archives/oai_spgwu_image_info.log'
            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-spgwu-tiny:' + spgwuTag + ' >> archives/oai_spgwu_image_info.log'
229
230
231
          } catch (Exception e) {
            error "OAI SPGW-U-Tiny Image tag to test does not exist!"
          }
232
233
234
        }
      }
    }
235
236
237
    stage ('Deploy Whole 5G Core Network') {
      steps {
        script {
238
          echo '\u2705 \u001B[32mDeploy CN5G using Docker-Compose\u001B[0m'
239
240
          // Prepare all needed files for docker-compose
          // First put all correct tags to test
241
          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" -e "s#AMF_IMAGE_TAG#' + amfTag + '#" -e "s#SMF_IMAGE_TAG#' + smfTag + '#" -e "s#SPGWU_IMAGE_TAG#' + spgwuTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
242
243
244
245
246
247
248
249
250
251
          dir('ci-scripts/dsTesterDockerCompose') {
            sh 'docker-compose up -d  > ../../archives/compose_5gcn_up.log 2>&1'
            sh 'sleep 100'
            // Do a check on number of healthy containers
            // 5 == mysql + nrf + amf + smf + upf(spgwu-tiny)
            ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
            ret = ret.trim()
            if (ret != '5') {
              error "Deployment went wrong!"
            }          
252
253
          }
        }
254
255
256
257
258
259
      }
      post {
        always {
          script {
            // Check status on mysql.
            try {
260
              sh 'docker exec cicd-mysql-svr /bin/bash -c "mysqladmin -u root --password=linux ping" > archives/mysql_status.log 2>&1'
261
262
263
              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-mysql-svr >> archives/mysql_status.log'
            } catch (Exception e) {
              sh 'echo "STATUS: KO" >> archives/mysql_status.log'
264
            }
265
266
            // Do docker logs to recover the configuration results
            try {
267
              //sh 'docker logs cicd-oai-nrf > archives/nrf_config.log 2>&1'
268
269
270
271
272
              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf >> archives/nrf_config.log'
            } catch (Exception e) {
              sh 'echo "STATUS: KO" >> archives/nrf_config.log'
            }
            try {
273
              //sh 'docker logs cicd-oai-amf > archives/amf_config.log 2>&1'
274
275
276
277
278
              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-amf >> archives/amf_config.log'
            } catch (Exception e) {
              sh 'echo "STATUS: KO" >> archives/amf_config.log'
            }
            try {
279
              //sh 'docker logs cicd-oai-smf > archives/smf_config.log 2>&1'
280
281
282
283
284
              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-smf >> archives/smf_config.log'
            } catch (Exception e) {
              sh 'echo "STATUS: OK" >> archives/smf_config.log'
            }
            try {
285
              //sh 'docker logs cicd-oai-upf > archives/spgwu_config.log 2>&1'
286
287
288
              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-upf >> archives/spgwu_config.log'
            } catch (Exception e) {
              sh 'echo "STATUS: KO" >> archives/spgwu_config.log'
289
290
291
            }
          }
        }
292
293
294
        success {
          script {
            sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log'
Raphael Defosseux's avatar
Raphael Defosseux committed
295
            sh 'python3 ./ci-scripts/routeCheck.py --mode=Add --userName=' + dsT_host_user + ' --hostName=' + dsT_host
296
297
298
299
300
301
302
          }
        }
        unsuccessful {
          script {
            sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log'
          }
        }
303
304
305
306
307
308
309
      }
    }
    stage ('Check with DS Tester') {
      when { expression {dsT_host_flag} }
      steps {
        lock (ds_tester_ci_resource) {
          script {
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
            echo '\u2705 \u001B[32mTesting with DS Tester\u001B[0m'
            if (fileExists("dstester")) {
              sh "rm -Rf dstester > /dev/null 2>&1"
            }
            sh "mkdir -p dstester"
            dir ('dstester') {
              withCredentials([
                [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.dsTesterGitLabRepository_Credentials}", usernameVariable: 'git_username', passwordVariable: 'git_token']
              ]) {
                sh "git clone https://${git_username}:${git_token}@github.com/OPENAIRINTERFACE/chasseur.git . > ../git_clone.log 2>&1"
                sh "git checkout develop >> ../git_clone.log 2>&1"
                try {
                  sh "python3 jenkins/dogmatix-agent.py -f jenkins/suit-docker-compose.yaml | tee ../DS-TEST-RESULTS/dsTester_Summary.txt"
                } catch (Exception e) {
                  currentBuild.result = 'FAILURE'
                  echo "dsTester Running FAILED"
                } 
              }
            }
            sh "python3 ./ci-scripts/toCheckDSTesterResult1.py"
          }  
331
332
        }
      }
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
      post {
        always {
          script {
            // Copy the pcaps and logs from the containers
            sh "mkdir -p archives/pcaps archives/logs"
            try {
              sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap'
              sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log'
              sh 'docker cp cicd-oai-amf:/tmp/amf.pcap archives/pcaps/oai_amf.pcap'
              sh 'docker cp cicd-oai-amf:/tmp/amf.log archives/logs/oai_amf.log'
              sh 'docker cp cicd-oai-smf:/tmp/smf.pcap archives/pcaps/oai_smf.pcap'
              sh 'docker cp cicd-oai-smf:/tmp/smf.log archives/logs/oai_smf.log'
              sh 'docker cp cicd-oai-upf:/tmp/spgwu.pcap archives/pcaps/oai_spgwu.pcap'
              sh 'docker cp cicd-oai-upf:/tmp/spgwu.log archives/logs/oai_spgwu.log'
            } catch (Exception e) {
              sh 'echo "Error in copying pcaps & logs from the containers"'
            }
          }
        }
      }
353
    }
354
355
356
357
    stage ('Undeploy 5G-CN') {
      steps {
        script {
          echo '\u2705 \u001B[32mUn-Deploy CN5G\u001B[0m'
Raphael Defosseux's avatar
Raphael Defosseux committed
358
          sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
359
360
          dir('ci-scripts/dsTesterDockerCompose') {
            sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1'
361
362
363
364
365
366
367
368
          }
        }
      }
    }
  }
  post {
    always {
      script { 
369
        // Remove any leftover containers/networks
Raphael Defosseux's avatar
Raphael Defosseux committed
370
        sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
371
372
373
374
        dir('ci-scripts/dsTesterDockerCompose') {
          sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1'
        }
        // Generating the HTML report
375
        sh 'python3 ./ci-scripts/dsTestGenerateHTMLReport2.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
376

377
        // Zipping all archived log files
378
        sh "zip -r -qq cn5g_fed_docker_logs.zip archives DS-TEST-RESULTS"
379
380
        if (fileExists('cn5g_fed_docker_logs.zip')) {
          archiveArtifacts artifacts: 'cn5g_fed_docker_logs.zip'
381
        }
382
383
        if (fileExists('test_results_oai_cn5g.html')) {
          archiveArtifacts artifacts: 'test_results_oai_cn5g.html'
384
        }
385
386
387
388
389
390
391
392
393
      }
    }
  }
}

// Functions

def copyTo2ndServer(filename, flag, user, host) {
  if (flag) {
394
395
396
397
398
399
    if ("oai-cn5g-fed.zip".equals(filename)) {
      sh "ssh ${user}@${host} 'rm -rf /tmp/CI-CN5G-FED'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN5G-FED'"
    }
    sh "scp ${filename} ${user}@${host}:/tmp/CI-CN5G-FED"
    if ("oai-cn5g-fed.zip".equals(filename)) {
Raphael Defosseux's avatar
Raphael Defosseux committed
400
      sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED && unzip -qq oai-cn5g-fed.zip && rm oai-cn5g-fed.zip'"
401
402
      sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED && git checkout -f ${GIT_COMMIT}'"
      sh "ssh ${user}@${host} 'cd /tmp/CI-CN5G-FED && git log -n1'"
403
    }
404
405
406
407
408
  }
}

def copyFrom2ndServer(filename, target, flag, user, host) {
  if (flag) {
409
    sh "scp ${user}@${host}:/tmp/CI-CN5G-FED/${filename} ${target}"
410
411
412
413
414
  }
}

def myShCmd(cmd, flag, user, host) {
  if (flag) {
415
    sh "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN5G-FED && ${cmd}'"
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
  } else {
    sh "${cmd}"
  }
}

def myShCmdWithLog(cmd, logFile, flag, user, host) {
  if (flag) {
    sh "ssh -t -t ${user}@${host} 'export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:.:/usr/local/devsol/bin && ${cmd}' > ${logFile} 2>&1"
  } else {
    sh "${cmd} > ${logFile} 2>&1"
  }
}

def myShRetCmd(cmd, flag, user, host) {
  if (flag) {
431
    ret = sh returnStdout: true, script: "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN5G-FED && ${cmd}'"
432
433
434
435
436
437
  } else {
    ret = sh returnStdout: true, script: "${cmd}"
  }
  ret = ret.trim()
  return ret
}