GONRG-3411 add WITSML E2E tests

.gitlab-ci.yml

@@ -19,7 +19,7 @@ variables:
   OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_APPLICATION_NAME: energistics
-  OSDU_GCP_DATASET_BASE_URL: https://os-dataset-attcrcktoa-uc.a.run.app
+  OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
   OSDU_GCP_MOCK_BASE_URL: localhost
 
   WORKDIR: energistics
@@ -34,6 +34,7 @@ stages:
   - containerize
   - csp_containerize
   - deploy
+  - end_to_end_postman_tests
 
 
 build_base_image:
@@ -62,7 +63,8 @@ build_base_image:
 
 .gcp_update_ini_file: &gcp_update_ini_file
   - echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
-  - sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_DATASET_BASE_URL|" build/providers/gcp/osdu_api.ini
+  - sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
+  - sed -i "s|%(STORAGE_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
   - sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
 
 build_osdu_gcp_image:
@@ -146,3 +148,24 @@ osdu_gcp_dags_prod_rsync:
   only:
     - /^release\/*/
   when: manual
+
+gcp_e2e_test_dag:
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  stage: end_to_end_postman_tests
+  needs: [osdu_gcp_dags_rsync]
+  script:
+    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
+    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "COMMUNITY"
+  only:
+    - $CI_DEFAULT_BRANCH
+    - /^trusted-/
+
+gcp_e2e_test_dag_pre_prod:
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  stage: end_to_end_postman_tests
+  needs: [osdu_gcp_dags_prod_rsync]
+  script:
+    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
+    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "PREPROD" 
+  only:
+    - /^release\/*

energistics/src/witsml_parser/create_energistics_manifest.py

@@ -25,6 +25,7 @@ import requests
 import tenacity
 from osdu_api.auth.authorization import TokenRefresher
 from osdu_api.clients.dataset.dataset_dms_client import DatasetDmsClient
+from osdu_api.clients.storage.record_client import RecordClient
 from osdu_api.configuration.config_manager import DefaultConfigManager
 from osdu_ingestion.libs.context import Context
 from osdu_ingestion.libs.handle_file import FileHandler
@@ -234,7 +235,7 @@ class EnergisticsManifestCreator:
             dataset_dms_client = DatasetDmsClient(DefaultConfigManager(), self.payload_context.data_partition_id)
             bearer_token = None if dataset_dms_client.use_service_principal else self.token_refresher.refresh_token()
             retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"], bearer_token=bearer_token)
-            
+
             # logger.info(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
             print(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
             if retrieval_instructions_response.status_code == 200:
@@ -243,7 +244,12 @@ class EnergisticsManifestCreator:
                 print(f"Dataset retrieval response body: {retrieval_instructions_response.content}")
                 if len(retrieval_instructions["delivery"]) > 0:
                     download_signed_url = retrieval_instructions["delivery"][0]["retrievalProperties"]["signedUrl"]
-                    file_source = retrieval_instructions["delivery"][0]["retrievalProperties"]["unsignedUrl"]
+
+            storage_client = RecordClient(DefaultConfigManager(), self.payload_context.data_partition_id)
+            record_response = storage_client.get_specific_record(recordId=retrieval_instructions["delivery"][0]["datasetRegistryId"], version="", bearer_token=bearer_token)
+            record = json.loads(record_response.content)
+            file_source = record["data"]["DatasetProperties"]["FileSourceInfo"]["FileSource"]
+            print(f"Record's FileSource: {file_source}")
         else:
             preload_file_path = execution_context["Context"]["preload_file_path"]
             

tests/e2e_postman_tests/gcp/run_e2e_tests.sh

+# /usr/bin/sh
+
+apt-get update
+apt-get install jq -y
+apt-get install nodejs -y > /dev/null 2>&1
+node -v
+apt install npm -y > /dev/null 2>&1
+PATH="$PATH"
+npm install -g n
+n latest
+PATH="$PATH"
+npm install -g newman -y
+
+# clone collections
+mkdir platform_testing
+git clone --depth 1 -b master https://community.opengroup.org/osdu/platform/testing.git platform_testing
+
+COLLECTION="platform_testing/Postman Collection/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
+COLLECTION_FOLDER="platform_testing/Postman Collection/41_CICD_WITSML"
+
+case $1 in
+    "PREPROD") 
+        ENVIRONMENT="platform_testing/Postman Collection/00_CICD_Setup_Environment/gcp.OSDU R3 PROD v2.4.postman_environment.json"
+        ;;
+    "COMMUNITY")
+        ENVIRONMENT="tests/e2e_postman_tests/gcp/environments/OSDU_Community.postman_environment.json"
+        ;;
+    *)
+        echo "Env option must be 'PREPROD' or 'COMMUNITY'"
+        exit 1
+        ;;
+esac
+
+# Get access token for GCP SA.
+echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
+gcloud auth activate-service-account --key-file file.json
+gcloud config set project $OSDU_GCP_PROJECT
+export ACCESS_TOKEN=`gcloud auth print-access-token`
+
+# As we use acces_token of Integration Tester SA File, we must skip Refresh Token step
+cat "$COLLECTION" | jq 'del(.item[] | select (.name=="Configure collection"))' > collection.json
+cat collection.json > "$COLLECTION"
+
+
+# Wait for DAGs initialization
+sleep 120
+
+newman run "$COLLECTION" --working-dir "$COLLECTION_FOLDER" --env-var access_token="$ACCESS_TOKEN" -g "$ENVIRONMENT" || EXIT_CODE=$?
+
+
+exit ${EXIT_CODE}