Skip to content
GitLab
Commit 525eb1b8 authored by Kamlesh Todai's avatar Kamlesh Todai
Browse files

Merge branch 'master' into 'gcp_tests' 

# Conflicts:
#   .gitlab-ci.yml
#   generate-pipeline.py
parents 0e6e4e4f 96550a76
Pipeline #3905 failed with stages
in 8 seconds
Expand all Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 525eb1b8
......@@ -11,39 +11,17 @@ generate-pipeline:
stage: generate
artifacts:
paths:
- '*.gitlab-ci.yml'
- 'test-execution.gitlab-ci.yml'
script:
- ./generate-pipeline.py
- tail -n 10000 *.gitlab-ci.yml
- cat test-execution.gitlab-ci.yml
# --------------------------------------------------------------------------------
# Invoke a child pipeline per environment
# Child pipeline execution
aws v2.4:
execute-tests:
trigger:
strategy: depend
include:
- artifact: aws.OSDU R3 PROD v2.4.gitlab-ci.yml
- artifact: test-execution.gitlab-ci.yml
job: generate-pipeline
azure v2.4:
trigger:
strategy: depend
include:
- artifact: azure.OSDU R3 PROD v2.4.gitlab-ci.yml
job: generate-pipeline
gcp v2.4:
trigger:
strategy: depend
include:
- artifact: gcp.OSDU R3 PROD v2.4.gitlab-ci.yml
job: generate-pipeline
#ibm v2.4:
# trigger:
# strategy: depend
# include:
# - artifact: ibm.OSDU R3PROD v2.4.gitlab-ci.yml
# job: generate-pipeline
Postman Collection/12_CICD_Setup_StorageAPI/Storage API CI-CD v1.11.postman_collection.json
View file @ 525eb1b8
......@@ -309,7 +309,7 @@
],
"query": [
{
"key": "cursor",
"key": "Cursor",
"value": "aa",
"description": "string",
"disabled": true
......@@ -333,12 +333,27 @@
"script": {
"id": "88a8fa2c-37fa-44b9-8a11-019480bee886",
"exec": [
"pm.test(\"Status code is 500\", function () {",
"/*pm.test(\"Status code is 500\", function () {",
" pm.response.to.have.status(500);",
"});",
"pm.test(\"Status description - Unknown Error.\", function () {",
" pm.response.to.have.status(\"Unknown Error.\");",
"});"
"});*/",
"",
"pm.test(\"Status code to be 401 or 403\", function () {",
" pm.expect(pm.response.code).to.be.oneOf([401,403]);",
"});",
"try {",
" let resBody = pm.response.json();",
" if (resBody != null) {",
" pm.test(\"Response has not authorized string\", function () {",
" pm.expect(pm.response.text()).to.include(\"not authorized\");",
" });",
" }",
"}",
"catch (e) {",
" console.log(e.message);",
"}"
],
"type": "text/javascript"
}
......@@ -349,13 +364,13 @@
"header": [
{
"description": "This value should be the desired data partition id.",
"key": "data-partition-id",
"key": "slb-data-partition-id",
"type": "text",
"value": "{{data-partition-id}}"
}
],
"url": {
"raw": "https://{{STORAGE_HOST}}/query/kinds?cursor=&limit=100",
"raw": "https://{{STORAGE_HOST}}/query/kinds?Cursor=&limit=100",
"protocol": "https",
"host": [
"{{STORAGE_HOST}}"
......@@ -366,7 +381,7 @@
],
"query": [
{
"key": "cursor",
"key": "Cursor",
"value": "",
"description": "string"
},
......@@ -586,7 +601,7 @@
],
"query": [
{
"key": "cursor",
"key": "Cursor",
"value": "",
"description": "string",
"disabled": true
......@@ -798,7 +813,7 @@
],
"body": {
"mode": "raw",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]\n\n",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]\n\n",
"options": {
"raw": {
"language": "json"
......@@ -873,7 +888,7 @@
],
"body": {
"mode": "raw",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake update1\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]\n",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake update1\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]\n",
"options": {
"raw": {
"language": "json"
......@@ -936,7 +951,7 @@
],
"body": {
"mode": "raw",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroupInvalid}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{tagName}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroupInvalid}}@{{data-partition-id}}{{domain}}\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{tagName}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]",
"options": {
"raw": {
"language": "json"
......@@ -999,7 +1014,7 @@
],
"body": {
"mode": "raw",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroupInvalid}}@{{data-partition-id}}.lk\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]",
"raw": "[\n {\n \"acl\": {\n \"owners\": [\n \"{{New_OwnerDataGroupInvalid}}@{{data-partition-id}}.lk\"\n ],\n \"viewers\": [\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\n ]\n },\n \"data\": {\n \"msg\": \"hello world from Data Lake\"\n },\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\n \"legal\": {\n \"legaltags\": [\n \"{{LegalTagNameExists}}\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ],\n \"status\": \"compliant\"\n },\n \"meta\": [\n {}\n ],\n \"version\": 0\n }\n]",
"options": {
"raw": {}
}
......@@ -1061,7 +1076,7 @@
],
"body": {
"mode": "raw",
"raw": "[\r\n {\r\n \"acl\": {\r\n \"owners\": [\r\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\r\n ],\r\n \"viewers\": [\r\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\r\n ]\r\n },\r\n \"data\": {\r\n \"msg\": \"hello world from Data Lake\"\r\n },\r\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\r\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\r\n \"legal\": {\r\n \"legaltags\": [\r\n \"{{LegalTagNameExists}}\"\r\n ],\r\n \"otherRelevantDataCountries\": [\r\n \"US\"\r\n ],\r\n \"status\": \"compliant\"\r\n },\r\n \"meta\": [\r\n {}\r\n ],\r\n \"version\": 0\r\n }\r\n]\r\n\r\n",
"raw": "[\r\n {\r\n \"acl\": {\r\n \"owners\": [\r\n \"{{New_OwnerDataGroup}}@{{data-partition-id}}{{domain}}\"\r\n ],\r\n \"viewers\": [\r\n \"{{New_ViewerDataGroup}}@{{data-partition-id}}{{domain}}\"\r\n ]\r\n },\r\n \"data\": {\r\n \"msg\": \"hello world from Data Lake\"\r\n },\r\n \"id\": \"{{data-partition-id}}:osdu:{{NewWellUWI}}\",\r\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\r\n \"legal\": {\r\n \"legaltags\": [\r\n \"{{LegalTagNameExists}}\"\r\n ],\r\n \"otherRelevantDataCountries\": [\r\n \"US\"\r\n ],\r\n \"status\": \"compliant\"\r\n },\r\n \"meta\": [\r\n {}\r\n ],\r\n \"version\": 0\r\n }\r\n]\r\n\r\n",
"options": {
"raw": {}
}
......@@ -2147,7 +2162,7 @@
],
"body": {
"mode": "raw",
"raw": "{\n \"kindsinvalid\" : \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\n \"schema\" : [ {\n \"path\" : \"msg\",\n \"kind\" : \"string\",\n \"ext\" : { }\n }, {\n \"path\" : \"year\",\n \"kind\" : \"int\",\n \"ext\" : { }\n }, {\n \"path\" : \"weight\",\n \"kind\" : \"float\",\n \"ext\" : { }\n }, {\n \"path\" : \"location\",\n \"kind\" : \"core:dl:geopoint:1.0.0\",\n \"ext\" : { }\n }, {\n \"path\" : \"valid\",\n \"kind\" : \"bool\",\n \"ext\" : { }\n } ]\n}\n",
"raw": "{\n \"kindsinvalid\" : \"{{data-partition-id}}:osdu:{{schema_version}}\",\n \"schema\" : [ {\n \"path\" : \"msg\",\n \"kind\" : \"string\",\n \"ext\" : { }\n }, {\n \"path\" : \"year\",\n \"kind\" : \"int\",\n \"ext\" : { }\n }, {\n \"path\" : \"weight\",\n \"kind\" : \"float\",\n \"ext\" : { }\n }, {\n \"path\" : \"location\",\n \"kind\" : \"core:dl:geopoint:1.0.0\",\n \"ext\" : { }\n }, {\n \"path\" : \"valid\",\n \"kind\" : \"bool\",\n \"ext\" : { }\n } ]\n}\n",
"options": {
"raw": {}
}
......@@ -2201,7 +2216,7 @@
],
"body": {
"mode": "raw",
"raw": "{\r\n \"kind\": \"{{data-partition-id}}:osdu:well-master:{{schema_version}}\",\r\n \"schema\": [\r\n {\r\n \"path\": \"msg\",\r\n \"kind\": \"string\"\r\n },\r\n {\r\n \"path\": \"year\",\r\n \"kind\": \"int\"\r\n },\r\n {\r\n \"path\": \"weight\",\r\n \"kind\": \"float\"\r\n },\r\n {\r\n \"path\": \"location\",\r\n \"kind\": \"core:dl:geopoint:1.0.0\"\r\n },\r\n {\r\n \"path\": \"valid\",\r\n \"kind\": \"bool\"\r\n }\r\n ]\r\n}",
"raw": "{\r\n \"kind\": \"{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}\",\r\n \"schema\": [\r\n {\r\n \"path\": \"msg\",\r\n \"kind\": \"string\"\r\n },\r\n {\r\n \"path\": \"year\",\r\n \"kind\": \"int\"\r\n },\r\n {\r\n \"path\": \"weight\",\r\n \"kind\": \"float\"\r\n },\r\n {\r\n \"path\": \"location\",\r\n \"kind\": \"core:dl:geopoint:1.0.0\"\r\n },\r\n {\r\n \"path\": \"valid\",\r\n \"kind\": \"bool\"\r\n }\r\n ]\r\n}",
"options": {
"raw": {}
}
......@@ -2510,8 +2525,8 @@
"script": {
"id": "281a844b-1557-40ea-ac6b-5bd75c241ebd",
"exec": [
"pm.test(\"Status code is 400\", function () {",
" pm.response.to.have.status(400);",
"pm.test(\"Status code is 404\", function () {",
" pm.response.to.have.status(404);",
"});",
"/*pm.test(\"Status description - Schema not found for specified kind.\", function () {",
" pm.response.to.have.status(\"Schema not found for specified kind.\");",
......@@ -2545,7 +2560,7 @@
{
"id": "a50bb3a8-2707-46af-96ff-3b319b881c15",
"key": "kind",
"value": "{{data-partition-id}}:osdu:well-master:{{schema_version}}",
"value": "{{data-partition-id}}:osdu:{{record_id}}:{{schema_version}}",
"type": "string"
}
]
......
generate-pipeline.py
View file @ 525eb1b8
......@@ -5,61 +5,88 @@ import os
environments = []
collections = []
i = 0
for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
if "Postman Collection" in root:
for f in filenames:
def addEntry(array):
array.append({
'nameParts': os.path.basename(f).split('.')[0:-2],
'path': os.path.join(root, f)
})
if f.endswith('.postman_environment.json'): addEntry(environments);
if f.endswith('.postman_collection.json'): addEntry(collections);
# --------------------------------------------------------------------------------
# First, walk the project's "Postman Collection" directory looking for files
# that match particular file extensions. When found, sort them into the
# environments # and collections arrays.
for environ in environments:
platform = environ['nameParts'][0]
environName = '.'.join(environ['nameParts'])
i = 0;
collectionDir = os.path.join(os.environ['CI_PROJECT_DIR'], 'Postman Collection')
with open(environName + '.gitlab-ci.yml', 'w') as pipelineConfig:
stageName = platform + ' Test'
for root, dirnames, filenames in os.walk(collectionDir):
for f in filenames:
def addEntry(array):
array.append({
'nameParts': os.path.basename(f).split('.')[0:-2],
'path': os.path.join(root, f)
})
if f.endswith('.postman_environment.json'): addEntry(environments)
if f.endswith('.postman_collection.json'): addEntry(collections)
pipelineConfig.write('\n'.join([
'stages:',
' - ' + stageName,
'',
'.newman:',
' stage: ' + stageName,
' tags: ' + "[\"docker-runner\"]",
' image:',
' name: postman/newman_alpine33',
' entrypoint: [""]',
' variables:',
' ENVIRONMENT: ' + environ['path'],
' script:',
' - newman --version ',
'# - npm install -g newman-reporter-html',
'# - npm install -g newman-reporter-htmlextra',
'# - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
' - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$' + platform.upper() + '_TEST_COLLECTION_CONFIG" --reporters cli',
'',
'# --------------------------------------------------------------------------------'
]) + '\n')
# --------------------------------------------------------------------------------
# Now, output the test execution pipeline based on the inner join of these
# arrays (the combination of every environment with every collection)
for collect in collections:
collectionName = '.'.join(collect['nameParts'])
sep = '_';
collString = sep.join(collect['nameParts']);
platformString = platform + ".";
htmlReportName = platformString + collString + "_report_extra.html";
junitReportName = platformString + collString + "_report_junit.xml"
with open('test-execution.gitlab-ci.yml', 'w') as pipelineConfig:
pipelineConfig.write('\n'.join([
'',
collectionName + ':',
' extends: .newman',
' variables:',
' COLLECTION_JSON: "' + collect['path'] + '"',
' HTML_REPORT_NAME: "' + htmlReportName + '"',
' JUNIT_REPORT_NAME: "' + junitReportName + '"'
]) + '\n')
# Create a stage for each environment, so they have their own columns
pipelineConfig.write('stages:\n');
for environ in environments:
platform = environ['nameParts'][0]
pipelineConfig.write(' - ' + platform + '\n');
pipelineConfig.write('\n# --------------------------------------------------------------------------------\n\n');
# Output the fixed rule pattern for newman jobs
pipelineConfig.write('\n'.join([
'.newman:',
' tags: ' + "[\"docker-runner\"]",
' image:',
' name: postman/newman_alpine33',
' entrypoint: [""]',
' script:',
' - TEST_COLLECTION_CONFIG=$(eval echo \$$TEST_COLLECTION_CONFIG_VARNAME)',
' - |',
' test -f "$TEST_COLLECTION_CONFIG" || ((',
' echo TEST_COLLECTION_CONFIG does not appear to be a file.',
' echo This probably means that you are running on a non-protected branch and did not',
' echo provide the deployment secrets manually. Or it could be a misconfigured variable.',
' echo',
' echo Warning: I am intentionally not printing the value of the variable in case it is the secret content itself.',
' ) && false)',
' - newman --version ',
#' - npm install -g newman-reporter-html',
#' - npm install -g newman-reporter-htmlextra',
#' - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$TEST_COLLECTION_CONFIG" --reporters cli,htmlextra,junit --reporter-htmlextra-export "$HTML_REPORT_NAME" --reporter-junit-export "$JUNIT_REPORT_NAME"',
' - newman run "$COLLECTION_JSON" -e "$ENVIRONMENT" -d "$TEST_COLLECTION_CONFIG" --reporters cli',
''
]))
# Output the configured jobs
for environ in environments:
pipelineConfig.write('\n# --------------------------------------------------------------------------------\n');
platform = environ['nameParts'][0]
environName = '.'.join(environ['nameParts'])
for collect in collections:
collectionName = '.'.join(collect['nameParts'])
sep = '_'
collString = sep.join(collect['nameParts'])
platformString = platform + "."
htmlReportName = platformString + collString + "_report_extra.html"
junitReportName = platformString + collString + "_report_junit.xml"
pipelineConfig.write('\n'.join([
'',
collectionName + ' -- ' + platform + ':',
' extends: .newman',
' stage: ' + platform,
' needs: []',
' variables:',
' ENVIRONMENT: "' + environ['path'] + '"',
' TEST_COLLECTION_CONFIG_VARNAME: ' + platform.upper() + '_TEST_COLLECTION_CONFIG',
' COLLECTION_JSON: "' + collect['path'] + '"',
' HTML_REPORT_NAME: "' + htmlReportName + '"',
' JUNIT_REPORT_NAME: "' + junitReportName + '"'
]) + '\n')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment