Commit b68d69cc authored by David Diederich's avatar David Diederich
Browse files

Merge remote-tracking branch 'origin/master' into timeout-increment

parents 29756c40 72519fdb
Pipeline #9021 passed with stage
in 6 seconds
CD.jpg

133 KB

CD1.jpg

137 KB

# CI-CD Pipelines
# OSDU CI/CD Pipeline Overview
Find the most recent documentation build [here](http://osdu.pages.community.opengroup.org/platform/ci-cd-pipelines/)
Find the most recent documentation build [here](http://osdu.pages.community.opengroup.org/platform/ci-cd-pipelines/)
A common place for shared CI/CD Pipeline files, for use will all OSDU projects.
![alt text](CI-CD.png "CI/CD Pipeline")
\ No newline at end of file
[The OSDU Platform project](https://community.opengroup.org/osdu/platform) (including sub-projects) makes use of [GitLab's built-in CI/CD capabilities](https://docs.gitlab.com/ee/ci/introduction/) to provide a CI/CD pipeline that runs automatically on commits to the project. The stages in the pipeline include:
- Security scanning for known vulnerabilities (CVEs)
- Scanning for use of approved Licenses in submitted code and dependencies
- Correct attribution of the submitted material
- Unit tests
- Integration tests
## Common Practices / Definitions
**Continuous Integration**
* Developers merge/integrate the code to Master as soon as developer made a progress on the feature. Usually everyone commits to the Master every day.
* Continuous Integration assumes a high degree of tests which are automated into the software. we have good automated testing coverage as we progress we need to add more test cases.
* Continuous Integration refers to Integrating code, building and testing code (Unit and Integration tests) within the development environment.
**Continuous Delivery**
* Continuous Delivery means that you are able to do frequent deployments to a testing environment and/or production environment but may choose not to do it, due to various reasons (Ex: businesses not ready or OSDU applicatiions not ready)
**Continuous Deployment**
* Continuous Deployment means that every change goes through the pipeline and automatically (No manual approval) gets put into production, resulting in many production deployments every day.
## A graphical overview of the CI/CD Pipeline:
**Continuous Delivery is not ready and its just an idea for now but we will have to figure out how & where to run UAT and Performance tests.**
**Continuous Deployment might not be applicable for OSDU.**
![alt text](CD1.jpg "CI/CD Pipeline")
......@@ -63,6 +63,7 @@
INDEXER_HOST: $AWS_INDEXER_HOST
AWS_CLUSTER_NAME: $AWS_CLUSTER_NAME
ENTITLEMENTS_URL: $AWS_ENTITLEMENTS_URL
VIRTUAL_SERVICE_HOST_NAME: $AWS_API_GATEWAY_URL
.aws:
tags: ['osdu-medium']
......
......@@ -34,6 +34,7 @@
STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
......@@ -84,6 +85,9 @@
ENVIRONMENT: CLOUD
LEGAL_TAG: opendes-public-usa-dataset-7643990
OTHER_RELEVANT_DATA_COUNTRIES: US
# Delivery Section
LEGAL_HOST: $LEGAL_URL
DELIVERY_HOST: $DELIVERY_URL
#Schema
PRIVATE_TENANT1: $MY_TENANT
PRIVATE_TENANT2: tenant2
......
......@@ -93,6 +93,12 @@ ibm-deploy:
FINISHED_WORKFLOW_ID: $IBM_FINISHED_WORKFLOW_ID
WORKFLOW_HOST: $IBM_WORKFLOW_HOST
# File
FILE_SERVICE_HOST: $IBM_FILE_SERVICE_HOST
DATA_PARTITION_ID: $IBM_DATA_PARTITION_ID
# Unit
VIRTUAL_SERVICE_HOST_NAME: $IBM_UNIT_SERVICE_HOST
ibm-test:
stage: integration
......@@ -118,3 +124,28 @@ ibm-test:
except:
variables:
- $IBM_SKIP_TEST == 'true'
ibm-test-py:
stage: integration
image: python:3.6
needs: ["ibm-deploy"]
extends:
- .ibm_variables
script:
- pip install --upgrade pip
- cd $IBM_INT_TEST_PY_SUBDIR
- pip install -r requirements.txt
- python ${IBM_INT_TEST_PY_FILE:-test.py}
# artifacts:
# when: always
# paths:
# - $IBM_INT_TEST_SUBDIR/target/*/TEST-*.xml
# reports:
# junit:
# - $IBM_INT_TEST_SUBDIR/target/*/TEST-*.xml
only:
variables:
- $IBM_INT_TEST_PY_SUBDIR && $IBM == 'true'
except:
variables:
- $IBM_SKIP_TEST == 'true'
......@@ -25,12 +25,18 @@
.osdu-gcp-variables:
variables:
#services' urls
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_AUTHORIZE_API: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_AUTHORIZE_API: $OSDU_GCP_ENTITLEMENTS_URL
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_RECORDS_ROOT_URL: $OSDU_GCP_STORAGE_URL
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1/
OSDU_GCP_AIRFLOW_URL: https://c410714dc037b0e02p-tp.appspot.com
OSDU_GCP_DELIVERY_SERVICE_URL: https://os-file-attcrcktoa-uc.a.run.app
OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app
OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app
OSDU_GCP_SEARCH_HOST: https://os-search-dot-nice-etching-277309.uc.r.appspot.com/api/search/v2/
OSDU_GCP_SEARCH_QUERY_URL: https://os-search-dot-nice-etching-277309.uc.r.appspot.com//api/search/v2/query
OSDU_GCP_INDEXER_HOST: https://os-indexer-dot-nice-etching-277309.uc.r.appspot.com/
# env parameters
OSDU_GCP_INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
OSDU_GCP_DEPLOY_ENV: dev
......@@ -44,13 +50,26 @@
# common integration tests variables
GOOGLE_AUDIENCE: $OSDU_GCP_INTEGRATION_TEST_AUDIENCE
DOMAIN: $OSDU_GCP_DOMAIN
INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
OTHER_RELEVANT_DATA_COUNTRIES: US
DEFAULT_DATA_PARTITION_ID_TENANT1: $OSDU_GCP_TENANT_NAME
LEGAL_TAG: osdu-demo-legaltag
GCLOUD_PROJECT: $OSDU_GCP_PROJECT
# integration test variables for ingestion workflow
WORKFLOW_HOST: $OSDU_GCP_WORKFLOW_SERVICE_URL
FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969
TEST_FILE_ID: $OSDU_GCP_TEST_FILE_ID
# integration test variables for ingestion service
DEFAULT_DATA_PARTITION_ID_TENANT1: $OSDU_GCP_TENANT_NAME
DEFAULT_DATA_PARTITION_ID_TENANT2: $OSDU_GCP_TENANT_NAME
INGEST_HOST: $OSDU_GCP_INGEST_HOST
TEST_OSDU_FILE_PATH: gs://nice-etching-277309-file/r1/data/provided/well-logs/1013_akm11_1978_comp.las
# integration test variables for delivery service
STORAGE_HOST: $OSDU_GCP_STORAGE_URL
ENTITLEMENTS_DOMAIN: $OSDU_GCP_DOMAIN
TENANT: $OSDU_GCP_TENANT_NAME
SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
INTEGRATION_TEST_AUDIENCE: $OSDU_GCP_INTEGRATION_TEST_AUDIENCE
SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
DELIVERY_HOST: $OSDU_GCP_DELIVERY_SERVICE_URL
osdu-gcp-containerize:
stage: containerize
......@@ -89,6 +108,8 @@ osdu-gcp-test:
needs: ["osdu-gcp-deploy"]
allow_failure: true
script:
- export INTEGRATION_TESTER=$(echo $OSDU_GCP_R3_INTEGRATION_TESTER)
- export NO_DATA_ACCESS_TESTER=$(echo $OSDU_GCP_R3_NO_DATA_ACCESS_TESTER)
- export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
- mvn --batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml -f testing/pom.xml verify --projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-gcp
only:
......
.OSDU_GCP_variables:
variables:
# service's URLs
OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_AUTHORIZE_API: $OSDU_GCP_ENTITLEMENTS_URL
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_RECORDS_ROOT_URL: $OSDU_GCP_STORAGE_URL
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1/
OSDU_GCP_AIRFLOW_URL: https://c410714dc037b0e02p-tp.appspot.com
OSDU_GCP_DELIVERY_SERVICE_URL: https://os-file-attcrcktoa-uc.a.run.app
OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app
OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app
OSDU_GCP_SEARCH_HOST: https://os-search-dot-nice-etching-277309.uc.r.appspot.com/api/search/v2/
OSDU_GCP_SEARCH_QUERY_URL: https://os-search-dot-nice-etching-277309.uc.r.appspot.com//api/search/v2/query
OSDU_GCP_INDEXER_HOST: https://os-indexer-dot-nice-etching-277309.uc.r.appspot.com/
# variables for integration tests
OSDU_GCP_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
OSDU_GCP_NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
OSDU_GCP_PUBSUB_TOKEN: $OSDU_GCP_PUBSUB_TOKEN
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_DEPLOY_ENV: empty
OSDU_GCP_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_TENANT_NAME: osdu
#Old OSDU_GCP_AUTHORIZE_API: https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1/
OSDU_GCP_AUTHORIZE_API: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1
OSDU_GCP_MY_TENANT_PROJECT: osdu
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1/
OSDU_GCP_MY_TENANT_PROJECT: $OSDU_GCP_TENANT_NAME
OSDU_GCP_MY_TENANT: $OSDU_GCP_TENANT_NAME
OSDU_GCP_INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
OSDU_GCP_MY_TENANT: osdu
#Old OSDU_GCP_ENTITLEMENT_URL: https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1/
OSDU_GCP_ENTITLEMENT_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1
OSDU_GCP_VENDOR: gcp
OSDU_GCP_PROJECT: nice-etching-277309
# integration test variables for search service
ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
ENTITLEMENTS_HOST: $OSDU_GCP_ENTITLEMENTS_URL
DEFAULT_DATA_PARTITION_ID_TENANT1: $OSDU_GCP_TENANT_NAME
DEFAULT_DATA_PARTITION_ID_TENANT2: $OSDU_GCP_TENANT_NAME
ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD
ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER
ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST
ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT
SEARCH_ON_BEHALF_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
GCLOUD_PROJECT: $OSDU_GCP_PROJECT
INDEXER_HOST: $OSDU_GCP_INDEXER_HOST
DATA_GROUP: $OSDU_GCP_TENANT_NAME
LEGAL_TAG: osdu-demo-legaltag
OTHER_RELEVANT_DATA_COUNTRIES: US
INTEGRATION_TEST_AUDIENCE: $OSDU_GCP_INTEGRATION_TEST_AUDIENCE
SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
.osdu-gcp:
tags: ["osdu-medium"]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment