Draft: Repair cluster-build pipeline

The introduction of building RHEL8 images on the cluster in MR !1681 (merged) had a flaw: it would recreate image streams on every CI run, thereby losing the association to images in the image registry. In the CI, we change from time to time the ran-base image between tags develop and ci-temp. By recreating the image streams, we quickly lost the association of ran-base:develop, which the CI assumed to "just be there". In this case, most builds would fail, basically bringing the CI to a halt. The actually important change here is in 83736340

Marking as BUILD-ONLY as we cannot test this in the CI: the CI will trigger the normal RHEL8 build on asterix. I will reproduce the scenario we had previously, i.e., trigger this pipeline multiple times in a way that builds will change the ran-base image between ran-base:develop and ran-base:ci-temp. The expected outcome is to always have a successful build, of course.

@lthomas I have additional changes to not use the SSHConnection class, but this will come later. The same with the cleanup of the physims.

ci-scripts/cls_cluster.py

@@ -98,11 +98,13 @@ class Cluster:
 			if re.search(f'imagestream.image.openshift.io/{name} created', before) is None:
 				logging.error('error while creating imagestream: ' + sshSession.getBefore())
 				return False
+		else:
+			logging.debug(f'-> imagestream {name} found')
 		image = f'{name}:{newTag}'
 		sshSession.command(f'oc delete istag {image}', '\$', 5) # we don't care if this fails, e.g., if it is missing
 		sshSession.command(f'oc create istag {image}', '\$', 5)
 		before = sshSession.getBefore()
-		if re.search(f'imagestreamtag.image.openshift.io/{image} created ', before) is not None:
+		if re.search(f'imagestreamtag.image.openshift.io/{image} created', before) is not None:
 			return True
 		logging.error('error while creating imagestreamtag: ' + sshSession.getBefore())
 		return False
@@ -132,7 +134,7 @@ class Cluster:
 		while timeout_sec > 0:
 			# check status
 			for j in jobs:
-				sshSession.command(f'oc get pods | grep {j}', '\$', 5, silent = True)
+				sshSession.command(f'oc get pods | grep {j}', '\$', 10, silent = True)
 				if sshSession.getBefore().count('Completed') > 0: jobs.remove(j)
 				if sshSession.getBefore().count('Error') > 0:
 					logging.error(f'error for job {j}: ' + sshSession.getBefore())
@@ -236,7+238,7 @@
 		# logging to OC Cluster and then switch to corresponding project
 		mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}', '\$', 31)
 		if mySSH.getBefore().count('Login successful.') == 0:
 			logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
 			mySSH.close()
 			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
 			return False
@@ -258,7+260,7 @@
 			self._recreate_bc(mySSH, 'ran-base', baseTag, 'openshift/ran-base-bc.yaml')
 			ranbase_job = self._start_build(mySSH, 'ran-base')
 			attemptedImages += ['ran-base']
 			status = ranbase_job is not None and self._wait_build_end(mySSH, [ranbase_job], 600)
 			if not status: logging.error('failure during build of ran-base')
 			mySSH.command(f'oc logs {ranbase_job} > cmake_targets/log/ran-base.log', '\$', 10)
 			# recover logs by mounting image
@@ -343,7+345,7 @@
 				imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
 			logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
 
+		grep_exp = "\|".join(attemptedImages)
+		mySSH.command(f'oc get images | grep -e \'{grep_exp}\' > cmake_targets/log/image_registry.log', '\$', 10);
+
 		build_log_name = f'build_log_{self.testCase_id}'
 		cls_containerize.CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD)