Commit be73e2de authored by Siarhei Khaletski (EPAM)'s avatar Siarhei Khaletski (EPAM) 🚩
Browse files

Merge branch 'GONRG-3411_Add_E2E_tests' into 'master'

GONRG-3411 add WITSML E2E tests

See merge request !36
parents 0c89942e e52311de
Pipeline #74245 skipped with stages
......@@ -19,7 +19,7 @@ variables:
OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: energistics
OSDU_GCP_DATASET_BASE_URL: https://os-dataset-attcrcktoa-uc.a.run.app
OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_MOCK_BASE_URL: localhost
WORKDIR: energistics
......@@ -34,6 +34,7 @@ stages:
- containerize
- csp_containerize
- deploy
- end_to_end_postman_tests
build_base_image:
......@@ -62,7 +63,8 @@ build_base_image:
.gcp_update_ini_file: &gcp_update_ini_file
- echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
- sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_DATASET_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%(STORAGE_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
build_osdu_gcp_image:
......@@ -146,3 +148,24 @@ osdu_gcp_dags_prod_rsync:
only:
- /^release\/*/
when: manual
gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
needs: [osdu_gcp_dags_rsync]
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "COMMUNITY"
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
gcp_e2e_test_dag_pre_prod:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
needs: [osdu_gcp_dags_prod_rsync]
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "PREPROD"
only:
- /^release\/*
......@@ -25,6 +25,7 @@ import requests
import tenacity
from osdu_api.auth.authorization import TokenRefresher
from osdu_api.clients.dataset.dataset_dms_client import DatasetDmsClient
from osdu_api.clients.storage.record_client import RecordClient
from osdu_api.configuration.config_manager import DefaultConfigManager
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.handle_file import FileHandler
......@@ -234,7 +235,7 @@ class EnergisticsManifestCreator:
dataset_dms_client = DatasetDmsClient(DefaultConfigManager(), self.payload_context.data_partition_id)
bearer_token = None if dataset_dms_client.use_service_principal else self.token_refresher.refresh_token()
retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"], bearer_token=bearer_token)
# logger.info(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
print(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
if retrieval_instructions_response.status_code == 200:
......@@ -243,7 +244,12 @@ class EnergisticsManifestCreator:
print(f"Dataset retrieval response body: {retrieval_instructions_response.content}")
if len(retrieval_instructions["delivery"]) > 0:
download_signed_url = retrieval_instructions["delivery"][0]["retrievalProperties"]["signedUrl"]
file_source = retrieval_instructions["delivery"][0]["retrievalProperties"]["unsignedUrl"]
storage_client = RecordClient(DefaultConfigManager(), self.payload_context.data_partition_id)
record_response = storage_client.get_specific_record(recordId=retrieval_instructions["delivery"][0]["datasetRegistryId"], version="", bearer_token=bearer_token)
record = json.loads(record_response.content)
file_source = record["data"]["DatasetProperties"]["FileSourceInfo"]["FileSource"]
print(f"Record's FileSource: {file_source}")
else:
preload_file_path = execution_context["Context"]["preload_file_path"]
......
# /usr/bin/sh
apt-get update
apt-get install jq -y
apt-get install nodejs -y > /dev/null 2>&1
node -v
apt install npm -y > /dev/null 2>&1
PATH="$PATH"
npm install -g n
n latest
PATH="$PATH"
npm install -g newman -y
# clone collections
mkdir platform_testing
git clone --depth 1 -b master https://community.opengroup.org/osdu/platform/testing.git platform_testing
COLLECTION="platform_testing/Postman Collection/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
COLLECTION_FOLDER="platform_testing/Postman Collection/41_CICD_WITSML"
case $1 in
"PREPROD")
ENVIRONMENT="platform_testing/Postman Collection/00_CICD_Setup_Environment/gcp.OSDU R3 PROD v2.4.postman_environment.json"
;;
"COMMUNITY")
ENVIRONMENT="tests/e2e_postman_tests/gcp/environments/OSDU_Community.postman_environment.json"
;;
*)
echo "Env option must be 'PREPROD' or 'COMMUNITY'"
exit 1
;;
esac
# Get access token for GCP SA.
echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
gcloud auth activate-service-account --key-file file.json
gcloud config set project $OSDU_GCP_PROJECT
export ACCESS_TOKEN=`gcloud auth print-access-token`
# As we use acces_token of Integration Tester SA File, we must skip Refresh Token step
cat "$COLLECTION" | jq 'del(.item[] | select (.name=="Configure collection"))' > collection.json
cat collection.json > "$COLLECTION"
# Wait for DAGs initialization
sleep 120
newman run "$COLLECTION" --working-dir "$COLLECTION_FOLDER" --env-var access_token="$ACCESS_TOKEN" -g "$ENVIRONMENT" || EXIT_CODE=$?
exit ${EXIT_CODE}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment