Merge branch 'consolidated-pipeline-generation' into 'master'

Consolidated pipeline generation

See merge request !53

.gitlab-ci.yml

@@ -11,39 +11,17 @@ generate-pipeline:
   stage: generate
   artifacts:
     paths:
-      - '*.gitlab-ci.yml'
+      - 'test-execution.gitlab-ci.yml'
   script:
     - ./generate-pipeline.py
-    - tail -n 10000 *.gitlab-ci.yml
+    - cat test-execution.gitlab-ci.yml
 
 # --------------------------------------------------------------------------------
-# Invoke a child pipeline per environment
+# Child pipeline execution
 
-aws v2.4:
+execute-tests:
   trigger:
     strategy: depend
     include:
-      - artifact: aws.OSDU R3 PROD v2.4.gitlab-ci.yml
+      - artifact: test-execution.gitlab-ci.yml
         job: generate-pipeline
-
-azure v2.4:
-  trigger:
-    strategy: depend
-    include:
-      - artifact: azure.OSDU R3 PROD v2.4.gitlab-ci.yml
-        job: generate-pipeline
-
-#gcp v2.4:
-#  trigger:
-#    strategy: depend
-#    include:
-#      - artifact: gcp.OSDU R3 PROD v2.4.gitlab-ci.yml
-#        job: generate-pipeline
-
-#ibm v2.4:
-#  trigger:
-#    strategy: depend
-#    include:
-#      - artifact: ibm.OSDU R3PROD v2.4.gitlab-ci.yml
-#        job: generate-pipeline
-

generate-pipeline.py

@@ -5,62 +5,78 @@ import os
 environments = []
 collections = []
 
+# --------------------------------------------------------------------------------
+# First, walk the project's "Postman Collection" directory looking for files
+# that match particular file extensions. When found, sort them into the
+# environments # and collections arrays.
+
 i = 0;
-for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
-	if  "Postman Collection" in root:
-		for f in filenames:
-			def addEntry(array):
-				array.append({
-					'nameParts': os.path.basename(f).split('.')[0:-2],
-					'path': os.path.join(root, f)
-				})
-			if f.endswith('.postman_environment.json'): addEntry(environments);
-			if f.endswith('.postman_collection.json'):  addEntry(collections);
+collectionDir = os.path.join(os.environ['CI_PROJECT_DIR'], 'Postman Collection')
+
+for root, dirnames, filenames in os.walk(collectionDir):
+	for f in filenames:
+		def addEntry(array):
+			array.append({
+				'nameParts': os.path.basename(f).split('.')[0:-2],
+				'path': os.path.join(root, f)
+			})
+		if f.endswith('.postman_environment.json'): addEntry(environments)
+		if f.endswith('.postman_collection.json'):  addEntry(collections)
+
+# --------------------------------------------------------------------------------
+# Now, output the test execution pipeline based on the inner join of these
+# arrays (the combination of every environment with every collection)
+
+with open('test-execution.gitlab-ci.yml', 'w') as pipelineConfig:
 
-for environ in environments:
-	platform = environ['nameParts'][0]
-	environName = '.'.join(environ['nameParts'])
+	# Create a stage for each environment, so they have their own columns
+	pipelineConfig.write('stages:\n');
+	for environ in environments:
+		platform = environ['nameParts'][0]
+		pipelineConfig.write('  - ' + platform + '\n');
 
-	with open(environName + '.gitlab-ci.yml', 'w') as pipelineConfig:
-		stageName = platform + ' Test'
+	pipelineConfig.write('\n# --------------------------------------------------------------------------------\n\n');
 
-		pipelineConfig.write('\n'.join([
-			'stages:',
-			'  - ' + stageName,
-			'',
-			'.newman:',
-            '  stage: ' + stageName,
-            '  tags: ' + "[\"docker-runner\"]",
-            '  image:',
-            '    name: postman/newman_alpine33',
-            '    entrypoint: [""]',
-            '  variables:',
-            '    ENVIRONMENT: ' + environ['path'],
-            '  script:',
-			'    - newman --version ',
-			'#    - npm install -g newman-reporter-html',
-			'#    - npm install -g newman-reporter-htmlextra',
-            '#    - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper()  + '_TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
-            '    - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper()  + '_TEST_COLLECTION_CONFIG" --reporters cli',
-			'',
-			'# --------------------------------------------------------------------------------'
-		]) + '\n')
+	# Output the fixed rule pattern for newman jobs
+	pipelineConfig.write('\n'.join([
+		'.newman:',
+		'  tags: ' + "[\"docker-runner\"]",
+		'  image:',
+		'    name: postman/newman_alpine33',
+		'    entrypoint: [""]',
+		'  script:',
+		'    - newman --version ',
+		'#    - npm install -g newman-reporter-html',
+		'#    - npm install -g newman-reporter-htmlextra',
+		'#    - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper()  + '_TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
+		'    - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper()  + '_TEST_COLLECTION_CONFIG" --reporters cli',
+		''
+	]))
+
+	# Output the configured jobs
+	for environ in environments:
+		pipelineConfig.write('\n# --------------------------------------------------------------------------------\n');
+
+		platform = environ['nameParts'][0]
+		environName = '.'.join(environ['nameParts'])
 
 		for collect in collections:
 			collectionName = '.'.join(collect['nameParts'])
-			sep = '_';
-			collString = sep.join(collect['nameParts']);
-			platformString = platform + ".";
-			htmlReportName = platformString + collString + "_report_extra.html";
+			sep = '_'
+			collString = sep.join(collect['nameParts'])
+			platformString = platform + "."
+			htmlReportName = platformString + collString + "_report_extra.html"
 			junitReportName = platformString + collString + "_report_junit.xml"
 
 			pipelineConfig.write('\n'.join([
 				'',
-				collectionName + ':',
+				collectionName + ' -- ' + platform + ':',
 				'  extends: .newman',
+				'  stage: ' + platform,
+				'  needs: []',
 				'  variables:',
+				'    ENVIRONMENT: "' + environ['path'] + '"',
 				'    COLLECTION_JSON: "' + collect['path'] + '"',
 				'    HTML_REPORT_NAME: "' + htmlReportName + '"',
 				'    JUNIT_REPORT_NAME: "' + junitReportName + '"'
 			]) + '\n')
-