Commit 069729cf authored by David Diederich's avatar David Diederich
Browse files

Switch to a single generated pipeline to avoid jamming issues

Launching multiple pipelines seems to bring about problems with some of the
pipelines being unable to launch. By creating only a single, more complicated
pipeline, I hope to sidestep this issue.
parent 673d1e46
Pipeline #3778 failed with stages
in 8 seconds
......@@ -11,39 +11,17 @@ generate-pipeline:
stage: generate
artifacts:
paths:
- '*.gitlab-ci.yml'
- 'test-execution.gitlab-ci.yml'
script:
- ./generate-pipeline.py
- tail -n 10000 *.gitlab-ci.yml
- cat test-execution.gitlab-ci.yml
# --------------------------------------------------------------------------------
# Invoke a child pipeline per environment
# Child pipeline execution
aws v2.4:
execute-tests:
trigger:
strategy: depend
include:
- artifact: aws.OSDU R3 PROD v2.4.gitlab-ci.yml
- artifact: test-execution.gitlab-ci.yml
job: generate-pipeline
azure v2.4:
trigger:
strategy: depend
include:
- artifact: azure.OSDU R3 PROD v2.4.gitlab-ci.yml
job: generate-pipeline
#gcp v2.4:
# trigger:
# strategy: depend
# include:
# - artifact: gcp.OSDU R3 PROD v2.4.gitlab-ci.yml
# job: generate-pipeline
#ibm v2.4:
# trigger:
# strategy: depend
# include:
# - artifact: ibm.OSDU R3PROD v2.4.gitlab-ci.yml
# job: generate-pipeline
......@@ -5,62 +5,77 @@ import os
environments = []
collections = []
# --------------------------------------------------------------------------------
# First, walk the project's "Postman Collection" directory looking for files
# that match particular file extensions. When found, sort them into the
# environments # and collections arrays.
i = 0;
for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
if "Postman Collection" in root:
for f in filenames:
def addEntry(array):
array.append({
'nameParts': os.path.basename(f).split('.')[0:-2],
'path': os.path.join(root, f)
})
if f.endswith('.postman_environment.json'): addEntry(environments);
if f.endswith('.postman_collection.json'): addEntry(collections);
collectionDir = os.path.join(os.environ['CI_PROJECT_DIR'], 'Postman Collection')
for root, dirnames, filenames in os.walk(collectionDir):
for f in filenames:
def addEntry(array):
array.append({
'nameParts': os.path.basename(f).split('.')[0:-2],
'path': os.path.join(root, f)
})
if f.endswith('.postman_environment.json'): addEntry(environments)
if f.endswith('.postman_collection.json'): addEntry(collections)
# --------------------------------------------------------------------------------
# Now, output the test execution pipeline based on the inner join of these
# arrays (the combination of every environment with every collection)
with open('test-execution.gitlab-ci.yml', 'w') as pipelineConfig:
for environ in environments:
platform = environ['nameParts'][0]
environName = '.'.join(environ['nameParts'])
# Create a stage for each environment, so they have their own columns
pipelineConfig.write('stages:\n');
for environ in environments:
platform = environ['nameParts'][0]
pipelineConfig.write(' - ' + platform + '\n');
with open(environName + '.gitlab-ci.yml', 'w') as pipelineConfig:
stageName = platform + ' Test'
pipelineConfig.write('\n# --------------------------------------------------------------------------------\n\n');
pipelineConfig.write('\n'.join([
'stages:',
' - ' + stageName,
'',
'.newman:',
' stage: ' + stageName,
' tags: ' + "[\"docker-runner\"]",
' image:',
' name: postman/newman_alpine33',
' entrypoint: [""]',
' variables:',
' ENVIRONMENT: ' + environ['path'],
' script:',
' - newman --version ',
'# - npm install -g newman-reporter-html',
'# - npm install -g newman-reporter-htmlextra',
'# - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
' - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli',
'',
'# --------------------------------------------------------------------------------'
]) + '\n')
# Output the fixed rule pattern for newman jobs
pipelineConfig.write('\n'.join([
'.newman:',
' tags: ' + "[\"docker-runner\"]",
' image:',
' name: postman/newman_alpine33',
' entrypoint: [""]',
' script:',
' - newman --version ',
'# - npm install -g newman-reporter-html',
'# - npm install -g newman-reporter-htmlextra',
'# - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
' - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli',
''
]))
# Output the configured jobs
for environ in environments:
pipelineConfig.write('\n# --------------------------------------------------------------------------------\n');
platform = environ['nameParts'][0]
environName = '.'.join(environ['nameParts'])
for collect in collections:
collectionName = '.'.join(collect['nameParts'])
sep = '_';
collString = sep.join(collect['nameParts']);
platformString = platform + ".";
htmlReportName = platformString + collString + "_report_extra.html";
sep = '_'
collString = sep.join(collect['nameParts'])
platformString = platform + "."
htmlReportName = platformString + collString + "_report_extra.html"
junitReportName = platformString + collString + "_report_junit.xml"
pipelineConfig.write('\n'.join([
'',
collectionName + ':',
platform + ' -- ' + collectionName + ':',
' extends: .newman',
' stage: ' + platform,
' variables:',
' ENVIRONMENT: "' + environ['path'] + '"',
' COLLECTION_JSON: "' + collect['path'] + '"',
' HTML_REPORT_NAME: "' + htmlReportName + '"',
' JUNIT_REPORT_NAME: "' + junitReportName + '"'
]) + '\n')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment