Added Data Load Scripts

docs/service-automation.md

@@ -54,7 +54,7 @@ az pipelines variable-group create \
   LEGAL_URL="https://${DNS_HOST}/api/legal/v1/" \
   STORAGE_URL="https://${DNS_HOST}/api/storage/v2/" \
   NO_DATA_ACCESS_TESTER='$(osdu-mvp-'${UNIQUE}'-noaccess-clientid)' \
-  NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET='$(osdu-mvp-'${UNIQUE}'-noaccess-secret))' \
+  NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET='$(osdu-mvp-'${UNIQUE}'-noaccess-secret)' \
   PUBSUB_TOKEN="az" \
   SERVICE_CONNECTION_NAME=$SERVICE_CONNECTION_NAME \
   GOOGLE_CLOUD_PROJECT="opendes" \
@@ -77,6 +77,7 @@ This variable group is a linked variable group that links to the Common Key Vaul
 - osdu-mvp-{unique}-application-oid
 - osdu-infra-{unique}-noaccess-clientid
 - osdu-infra-{unique}-noaccess-oid
+- osdu-infra-{unique}-noaccess-secret
 
 
 __Setup and Configure the ADO Library `Azure Target Env - demo`__

tools/rest/check.http

@@ -163,12 +163,6 @@ data-partition-id: {{data_partition_id}}
 }
 
 
-
-
-
-
-
-
 # -----------------------
 # API LEGAL
 # -----------------------
@@ -256,12 +250,6 @@ data-partition-id: {{data_partition_id}}
 @record_id = {{searchRecord.response.body.results[0].id}}
 
 
-###
-# @name searchElastic
-GET {{ES_HOST}}/{{index}}
-Authorization: Basic {{ES_AUTH_TOKEN}}
-Accept: application/json
-
 
 # -----------------------
 # CLEANUP

tools/rest/count.http

+# -------HTTP REST CLIENT -------
+# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+
+## This script provides a few samples for counting records.
+
+
+
+# -----------------------
+# OAUTH (Variables)
+# -----------------------
+###
+@login_base = login.microsoftonline.com/{{TENANT_ID}}
+@oauth_token_host = {{login_base}}/oauth2/v2.0/token
+@scopes = {{CLIENT_ID}}/.default openid profile offline_access
+
+
+
+# -----------------------
+# OAUTH  refresh_token
+# -----------------------
+###
+# @name refresh
+POST https://{{oauth_token_host}} HTTP/1.1
+Content-Type: application/x-www-form-urlencoded
+
+grant_type=refresh_token
+&client_id={{CLIENT_ID}}
+&client_secret={{CLIENT_SECRET}}
+&refresh_token={{INITIAL_TOKEN}}
+&scope={{scopes}}
+
+
+# -----------------------
+# API (Variables)
+# -----------------------
+###
+@access_token = {{refresh.response.body.access_token}}
+@ENDPOINT = https://{{OSDU_HOST}}
+@SEARCH_HOST = {{ENDPOINT}}/api/search/v2
+@data_partition_id = opendes
+@domain = osdu
+@tag = opendes-public-usa-dataset-1
+@schema_version = 0.2.0
+
+
+
+###
+# @name searchAll
+@all = {{data_partition_id}}:{{domain}}:*:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{all}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@all_totalCount = {{searchAll.response.body.totalCount}}
+
+
+###
+# @name searchWellMaster
+@wellmaster = {{data_partition_id}}:{{domain}}:well-master:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellmaster}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@wellmaster_SRN = {{searchWellMaster.response.body.results[0].data.ResourceID}}
+@wellmaster_id = {{searchWellMaster.response.body.results[0].id}}
+@wellmaster_totalCount = {{searchWellMaster.response.body.totalCount}}
+
+
+###
+# @name searchWellboreMaster
+@wellboremaster = {{data_partition_id}}:{{domain}}:wellbore-master:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellboremaster}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@wellmaster_SRN = {{searchWellboreMaster.response.body.results[0].data.ResourceID}}
+@wellmaster_id = {{searchWellboreMaster.response.body.results[0].id}}
+@wellmaster_totalCount = {{searchWellboreMaster.response.body.totalCount}}
+
+
+###
+# @name searchWelllog_wp
+@welllogwp = {{data_partition_id}}:{{domain}}:welllog-wp:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{welllogwp}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@welllogwp_SRN = {{searchWelllog_wp.response.body.results[0].data.ResourceID}}
+@welllogwp_ID = {{searchWelllog_wp.response.body.results[0].id}}
+@welllogwp_totalCount = {{searchWelllog_wp.response.body.totalCount}}
+
+###
+# @name searchWelllog_wpc
+@welllogwpc = {{data_partition_id}}:{{domain}}:welllog-wpc:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{welllogwpc}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@welllogwpc_SRN = {{searchWelllog_wpc.response.body.results[0].data.ResourceID}}
+@welllogwpc_id = {{searchWelllog_wpc.response.body.results[0].id}}
+@welllogwpc_totalCount = {{searchWelllog_wpc.response.body.totalCount}}
+
+
+###
+# @name searchWellboreMarker_wp
+@wellboremarkerwp = {{data_partition_id}}:{{domain}}:wellboremarker-wp:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellboremarkerwp}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@wellboremarkerwp_SRN = {{searchWellboreMarker_wp.response.body.results[0].data.ResourceID}}
+@wellboremarkerwp_ID = {{searchWellboreMarker_wp.response.body.results[0].id}}
+@wellboremarkerwp_totalCount = {{searchWellboreMarker_wp.response.body.totalCount}}
+
+
+###
+# @name searchWellboreMarker_wpc
+@wellboremarkerwpc = {{data_partition_id}}:{{domain}}:wellboremarker-wpc:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellboremarkerwpc}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@welllogwpc_SRN = {{searchWellboreMarker_wpc.response.body.results[0].data.ResourceID}}
+@welllogwpc_ID = {{searchWellboreMarker_wpc.response.body.results[0].id}}
+@welllogwpc_totalCount = {{searchWellboreMarker_wpc.response.body.totalCount}}
+
+
+###
+# @name searchWellboreTrajectory_wp
+@wellboretrajectorywp = {{data_partition_id}}:{{domain}}:wellboretrajectory-wp:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellboretrajectorywp}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@wellboretrajectorywp_SRN = {{searchWellboreTrajectory_wp.response.body.results[0].data.ResourceID}}
+@wellboretrajectorywp_ID = {{searchWellboreTrajectory_wp.response.body.results[0].id}}
+@wellboretrajectorywp_totalCount = {{searchWellboreTrajectory_wp.response.body.totalCount}}
+
+
+
+###
+# @name searchWellboreTrajectory_wpc
+@wellboretrajectorywpc = {{data_partition_id}}:{{domain}}:wellboretrajectory-wpc:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{wellboretrajectorywpc}}",
+  "offset": 0,
+  "limit": 1
+}
+
+@wellboretrajectorywp_SRN = {{searchWellboreTrajectory_wpc.response.body.results[0].data.ResourceID}}
+@wellboretrajectorywp_ID = {{searchWellboreTrajectory_wpc.response.body.results[0].id}}
+@wellboretrajectorywpc_totalCount = {{searchWellboreTrajectory_wpc.response.body.totalCount}}
+
+
+###
+# @name searchFile
+@file = {{data_partition_id}}:{{domain}}:file:{{schema_version}}
+POST {{SEARCH_HOST}}/query
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{file}}",
+  "offset": 0,
+  "limit": 3
+}
+
+@file_SRN = {{searchFile.response.body.results[0].data.ResourceID}}
+@file_ID = {{searchFile.response.body.results[0].id}}
+@file_totalCount = {{searchFile.response.body.totalCount}}

tools/rest/entitlement.http

+# -------HTTP REST CLIENT -------
+# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+
+## This script provides a few samples for calling entitlements.
+
+
+
+# -----------------------
+# OAUTH (Variables)
+# -----------------------
+###
+@login_base = login.microsoftonline.com/{{TENANT_ID}}
+@oauth_token_host = {{login_base}}/oauth2/v2.0/token
+@scopes = {{CLIENT_ID}}/.default openid profile offline_access
+
+
+
+# -----------------------
+# OAUTH  refresh_token
+# -----------------------
+###
+# @name refresh
+POST https://{{oauth_token_host}} HTTP/1.1
+Content-Type: application/x-www-form-urlencoded
+
+grant_type=refresh_token
+&client_id={{CLIENT_ID}}
+&client_secret={{CLIENT_SECRET}}
+&refresh_token={{INITIAL_TOKEN}}
+&scope={{scopes}}
+
+
+# -----------------------
+# API (Variables)
+# -----------------------
+###
+@access_token = {{refresh.response.body.access_token}}
+@ENDPOINT = https://{{OSDU_HOST}}
+@ENTITLEMENTS_HOST = {{ENDPOINT}}/entitlements/v1
+@data_partition_id = opendes
+
+
+
+# -----------------------
+# API: entitlements-azure
+# -----------------------
+
+
+###
+# @name myGroups
+GET {{ENTITLEMENTS_HOST}}/groups
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}
+
+
+###
+# @name user@email.com
+@new_user=<user_guid_here>
+POST {{ENTITLEMENTS_HOST}}/profile
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "id": "{{new_user}}",
+  "tenants": [{
+      "name": "common",
+      "groups": [
+          "service.storage.admin",
+          "service.legal.admin",
+          "data.datalake.admin",
+          "data.datalake.viewer",
+          "data.default.viewer",
+          "data.default.owner",
+          "service.search.admin",
+          "service.search.user",
+          "data.default.viewers",
+          "data.default.owners",
+          "service.entitlements.admin"
+      ]
+  }, {
+      "name": "opendes",
+      "groups": [
+          "service.storage.admin",
+          "service.legal.admin",
+          "data.test1",
+          "data.datalake.viewer",
+          "data.default.viewer",
+          "data.default.owner",
+          "service.search.admin",
+          "service.search.user",
+          "data.default.viewers",
+          "data.default.owners",
+          "service.entitlements.admin"
+      ]
+  }]
+}

tools/rest/legal.http

+# -------HTTP REST CLIENT -------
+# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+
+## This script provides a few samples for calling legal.
+
+
+
+# -----------------------
+# OAUTH (Variables)
+# -----------------------
+###
+@login_base = login.microsoftonline.com/{{TENANT_ID}}
+@oauth_token_host = {{login_base}}/oauth2/v2.0/token
+@scopes = {{CLIENT_ID}}/.default openid profile offline_access
+
+
+
+# -----------------------
+# OAUTH  refresh_token
+# -----------------------
+###
+# @name refresh
+POST https://{{oauth_token_host}} HTTP/1.1
+Content-Type: application/x-www-form-urlencoded
+
+grant_type=refresh_token
+&client_id={{CLIENT_ID}}
+&client_secret={{CLIENT_SECRET}}
+&refresh_token={{INITIAL_TOKEN}}
+&scope={{scopes}}
+
+
+# -----------------------
+# API (Variables)
+# -----------------------
+###
+@access_token = {{refresh.response.body.access_token}}
+@ENDPOINT = https://{{OSDU_HOST}}
+@LEGAL_HOST = {{ENDPOINT}}/api/legal/v1
+@data_partition_id = opendes
+@tag = opendes-public-usa-dataset-1
+
+
+# -----------------------
+# API: legal
+# -----------------------
+
+###
+# @name getAllTag
+GET {{LEGAL_HOST}}/legaltags
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}
+
+###
+# @name create_tag
+POST {{LEGAL_HOST}}/legaltags
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "name": "{{tag}}",
+  "description": "This tag is used by Data Upload Scripts",
+  "properties": {
+    "countryOfOrigin": [
+      "US"
+    ],
+    "contractId": "A1234",
+    "expirationDate": "2020-12-31",
+    "originator": "MyCompany",
+    "dataType": "Transferred Data",
+    "securityClassification": "Public",
+    "personalData": "No Personal Data",
+    "exportClassification": "EAR99"
+  }
+}
+
+###
+# @name getTag
+GET {{LEGAL_HOST}}/legaltags/{{tag}}
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}
+
+###
+# @name deleteTag
+DELETE {{LEGAL_HOST}}/legaltags/{{tag}}
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}

tools/rest/storage.http

+# -------HTTP REST CLIENT -------
+# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+
+## This script provides a few samples for calling storage.
+
+
+
+# -----------------------
+# OAUTH (Variables)
+# -----------------------
+###
+@login_base = login.microsoftonline.com/{{TENANT_ID}}
+@oauth_token_host = {{login_base}}/oauth2/v2.0/token
+@scopes = {{CLIENT_ID}}/.default openid profile offline_access
+
+
+
+# -----------------------
+# OAUTH  refresh_token
+# -----------------------
+###
+# @name refresh
+POST https://{{oauth_token_host}} HTTP/1.1
+Content-Type: application/x-www-form-urlencoded
+
+grant_type=refresh_token
+&client_id={{CLIENT_ID}}
+&client_secret={{CLIENT_SECRET}}
+&refresh_token={{INITIAL_TOKEN}}
+&scope={{scopes}}
+
+
+# -----------------------
+# API (Variables)
+# -----------------------
+###
+@access_token = {{refresh.response.body.access_token}}
+@ENDPOINT = https://{{OSDU_HOST}}
+@STORAGE_HOST = {{ENDPOINT}}/api/storage/v2
+@data_partition_id = opendes
+@kind = opendes:qa-test:well-master:0.2.0
+@id = <a_valid_document_id>
+
+# -----------------------
+# API: storage
+# -----------------------
+
+###
+# @name getAllSchema
+GET {{STORAGE_HOST}}/query/kinds
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}
+
+
+###
+# @name getAllSchemaOfKind
+GET {{STORAGE_HOST}}/query/kinds
+Authorization: Bearer {{access_token}}
+Accept: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+"kind": "opendes:osdu:*:*",
+"limit":1,
+"offset":0,
+"aggregateBy": "kind"
+}
+
+
+###
+# @name create_schema
+POST {{STORAGE_HOST}}/schemas
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "kind": "{{kind}}",
+  "schema": [
+    {
+      "path": "ResourceTypeID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceHomeRegionID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceHostRegionIDs",
+      "kind": "[]string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceObjectCreationDateTime",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceVersionCreationDateTime",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceCurationStatus",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceLifecycleStatus",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "ResourceSecurityClassification",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.FacilityID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.FacilityTypeID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.FacilityOperator.FacilityOperatorOrganisationID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.FacilityOperator.EffectiveDateTime",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.FacilityOperator.TerminationDateTime",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.DataSourceOrganisationID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.SpatialLocationCoordinatesDate",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.QuantitativeAccuracyBandID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.QualitativeSpatialAccuracyTypeID",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.CoordinateQualityCheckPerformedBy",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.CoordinateQualityCheckDateTime",
+      "kind": "string",
+      "ext": {}
+    },
+    {
+      "path": "Data.IndividualTypeProperties.SpatialLocation.CoordinateQualityCheckRemark",