Skip to content
GitLab
Commit a2c5b93d authored by Daniel Scholl's avatar Daniel Scholl
Browse files

Merge branch 'load_test' into 'master' 

Added Data Load Scripts

See merge request !42
parents 58c84ad5 8329cbe9
Expand all Hide whitespace changes
Inline Side-by-side
docs/service-automation.md
View file @ a2c5b93d
......@@ -54,7 +54,7 @@ az pipelines variable-group create \
LEGAL_URL="https://${DNS_HOST}/api/legal/v1/" \
STORAGE_URL="https://${DNS_HOST}/api/storage/v2/" \
NO_DATA_ACCESS_TESTER='$(osdu-mvp-'${UNIQUE}'-noaccess-clientid)' \
NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET='$(osdu-mvp-'${UNIQUE}'-noaccess-secret))' \
NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET='$(osdu-mvp-'${UNIQUE}'-noaccess-secret)' \
PUBSUB_TOKEN="az" \
SERVICE_CONNECTION_NAME=$SERVICE_CONNECTION_NAME \
GOOGLE_CLOUD_PROJECT="opendes" \
......@@ -77,6 +77,7 @@ This variable group is a linked variable group that links to the Common Key Vaul
- osdu-mvp-{unique}-application-oid
- osdu-infra-{unique}-noaccess-clientid
- osdu-infra-{unique}-noaccess-oid
- osdu-infra-{unique}-noaccess-secret
__Setup and Configure the ADO Library `Azure Target Env - demo`__
......
tools/rest/check.http
View file @ a2c5b93d
......@@ -163,12 +163,6 @@ data-partition-id: {{data_partition_id}}
}
# -----------------------
# API LEGAL
# -----------------------
......@@ -256,12 +250,6 @@ data-partition-id: {{data_partition_id}}
@record_id = {{searchRecord.response.body.results[0].id}}
###
# @name searchElastic
GET {{ES_HOST}}/{{index}}
Authorization: Basic {{ES_AUTH_TOKEN}}
Accept: application/json
# -----------------------
# CLEANUP
......
tools/rest/count.http 0 → 100644
View file @ a2c5b93d
# -------HTTP REST CLIENT -------
# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
## This script provides a few samples for counting records.
# -----------------------
# OAUTH (Variables)
# -----------------------
###
@login_base = login.microsoftonline.com/{{TENANT_ID}}
@oauth_token_host = {{login_base}}/oauth2/v2.0/token
@scopes = {{CLIENT_ID}}/.default openid profile offline_access
# -----------------------
# OAUTH refresh_token
# -----------------------
###
# @name refresh
POST https://{{oauth_token_host}} HTTP/1.1
Content-Type: application/x-www-form-urlencoded
grant_type=refresh_token
&client_id={{CLIENT_ID}}
&client_secret={{CLIENT_SECRET}}
&refresh_token={{INITIAL_TOKEN}}
&scope={{scopes}}
# -----------------------
# API (Variables)
# -----------------------
###
@access_token = {{refresh.response.body.access_token}}
@ENDPOINT = https://{{OSDU_HOST}}
@SEARCH_HOST = {{ENDPOINT}}/api/search/v2
@data_partition_id = opendes
@domain = osdu
@tag = opendes-public-usa-dataset-1
@schema_version = 0.2.0
###
# @name searchAll
@all = {{data_partition_id}}:{{domain}}:*:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{all}}",
"offset": 0,
"limit": 1
}
@all_totalCount = {{searchAll.response.body.totalCount}}
###
# @name searchWellMaster
@wellmaster = {{data_partition_id}}:{{domain}}:well-master:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellmaster}}",
"offset": 0,
"limit": 1
}
@wellmaster_SRN = {{searchWellMaster.response.body.results[0].data.ResourceID}}
@wellmaster_id = {{searchWellMaster.response.body.results[0].id}}
@wellmaster_totalCount = {{searchWellMaster.response.body.totalCount}}
###
# @name searchWellboreMaster
@wellboremaster = {{data_partition_id}}:{{domain}}:wellbore-master:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellboremaster}}",
"offset": 0,
"limit": 1
}
@wellmaster_SRN = {{searchWellboreMaster.response.body.results[0].data.ResourceID}}
@wellmaster_id = {{searchWellboreMaster.response.body.results[0].id}}
@wellmaster_totalCount = {{searchWellboreMaster.response.body.totalCount}}
###
# @name searchWelllog_wp
@welllogwp = {{data_partition_id}}:{{domain}}:welllog-wp:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{welllogwp}}",
"offset": 0,
"limit": 1
}
@welllogwp_SRN = {{searchWelllog_wp.response.body.results[0].data.ResourceID}}
@welllogwp_ID = {{searchWelllog_wp.response.body.results[0].id}}
@welllogwp_totalCount = {{searchWelllog_wp.response.body.totalCount}}
###
# @name searchWelllog_wpc
@welllogwpc = {{data_partition_id}}:{{domain}}:welllog-wpc:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{welllogwpc}}",
"offset": 0,
"limit": 1
}
@welllogwpc_SRN = {{searchWelllog_wpc.response.body.results[0].data.ResourceID}}
@welllogwpc_id = {{searchWelllog_wpc.response.body.results[0].id}}
@welllogwpc_totalCount = {{searchWelllog_wpc.response.body.totalCount}}
###
# @name searchWellboreMarker_wp
@wellboremarkerwp = {{data_partition_id}}:{{domain}}:wellboremarker-wp:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellboremarkerwp}}",
"offset": 0,
"limit": 1
}
@wellboremarkerwp_SRN = {{searchWellboreMarker_wp.response.body.results[0].data.ResourceID}}
@wellboremarkerwp_ID = {{searchWellboreMarker_wp.response.body.results[0].id}}
@wellboremarkerwp_totalCount = {{searchWellboreMarker_wp.response.body.totalCount}}
###
# @name searchWellboreMarker_wpc
@wellboremarkerwpc = {{data_partition_id}}:{{domain}}:wellboremarker-wpc:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellboremarkerwpc}}",
"offset": 0,
"limit": 1
}
@welllogwpc_SRN = {{searchWellboreMarker_wpc.response.body.results[0].data.ResourceID}}
@welllogwpc_ID = {{searchWellboreMarker_wpc.response.body.results[0].id}}
@welllogwpc_totalCount = {{searchWellboreMarker_wpc.response.body.totalCount}}
###
# @name searchWellboreTrajectory_wp
@wellboretrajectorywp = {{data_partition_id}}:{{domain}}:wellboretrajectory-wp:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellboretrajectorywp}}",
"offset": 0,
"limit": 1
}
@wellboretrajectorywp_SRN = {{searchWellboreTrajectory_wp.response.body.results[0].data.ResourceID}}
@wellboretrajectorywp_ID = {{searchWellboreTrajectory_wp.response.body.results[0].id}}
@wellboretrajectorywp_totalCount = {{searchWellboreTrajectory_wp.response.body.totalCount}}
###
# @name searchWellboreTrajectory_wpc
@wellboretrajectorywpc = {{data_partition_id}}:{{domain}}:wellboretrajectory-wpc:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{wellboretrajectorywpc}}",
"offset": 0,
"limit": 1
}
@wellboretrajectorywp_SRN = {{searchWellboreTrajectory_wpc.response.body.results[0].data.ResourceID}}
@wellboretrajectorywp_ID = {{searchWellboreTrajectory_wpc.response.body.results[0].id}}
@wellboretrajectorywpc_totalCount = {{searchWellboreTrajectory_wpc.response.body.totalCount}}
###
# @name searchFile
@file = {{data_partition_id}}:{{domain}}:file:{{schema_version}}
POST {{SEARCH_HOST}}/query
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{file}}",
"offset": 0,
"limit": 3
}
@file_SRN = {{searchFile.response.body.results[0].data.ResourceID}}
@file_ID = {{searchFile.response.body.results[0].id}}
@file_totalCount = {{searchFile.response.body.totalCount}}
tools/rest/entitlement.http 0 → 100644
View file @ a2c5b93d
# -------HTTP REST CLIENT -------
# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
## This script provides a few samples for calling entitlements.
# -----------------------
# OAUTH (Variables)
# -----------------------
###
@login_base = login.microsoftonline.com/{{TENANT_ID}}
@oauth_token_host = {{login_base}}/oauth2/v2.0/token
@scopes = {{CLIENT_ID}}/.default openid profile offline_access
# -----------------------
# OAUTH refresh_token
# -----------------------
###
# @name refresh
POST https://{{oauth_token_host}} HTTP/1.1
Content-Type: application/x-www-form-urlencoded
grant_type=refresh_token
&client_id={{CLIENT_ID}}
&client_secret={{CLIENT_SECRET}}
&refresh_token={{INITIAL_TOKEN}}
&scope={{scopes}}
# -----------------------
# API (Variables)
# -----------------------
###
@access_token = {{refresh.response.body.access_token}}
@ENDPOINT = https://{{OSDU_HOST}}
@ENTITLEMENTS_HOST = {{ENDPOINT}}/entitlements/v1
@data_partition_id = opendes
# -----------------------
# API: entitlements-azure
# -----------------------
###
# @name myGroups
GET {{ENTITLEMENTS_HOST}}/groups
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
###
# @name user@email.com
@new_user=<user_guid_here>
POST {{ENTITLEMENTS_HOST}}/profile
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"id": "{{new_user}}",
"tenants": [{
"name": "common",
"groups": [
"service.storage.admin",
"service.legal.admin",
"data.datalake.admin",
"data.datalake.viewer",
"data.default.viewer",
"data.default.owner",
"service.search.admin",
"service.search.user",
"data.default.viewers",
"data.default.owners",
"service.entitlements.admin"
]
}, {
"name": "opendes",
"groups": [
"service.storage.admin",
"service.legal.admin",
"data.test1",
"data.datalake.viewer",
"data.default.viewer",
"data.default.owner",
"service.search.admin",
"service.search.user",
"data.default.viewers",
"data.default.owners",
"service.entitlements.admin"
]
}]
}
tools/rest/legal.http 0 → 100644
View file @ a2c5b93d
# -------HTTP REST CLIENT -------
# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
## This script provides a few samples for calling legal.
# -----------------------
# OAUTH (Variables)
# -----------------------
###
@login_base = login.microsoftonline.com/{{TENANT_ID}}
@oauth_token_host = {{login_base}}/oauth2/v2.0/token
@scopes = {{CLIENT_ID}}/.default openid profile offline_access
# -----------------------
# OAUTH refresh_token
# -----------------------
###
# @name refresh
POST https://{{oauth_token_host}} HTTP/1.1
Content-Type: application/x-www-form-urlencoded
grant_type=refresh_token
&client_id={{CLIENT_ID}}
&client_secret={{CLIENT_SECRET}}
&refresh_token={{INITIAL_TOKEN}}
&scope={{scopes}}
# -----------------------
# API (Variables)
# -----------------------
###
@access_token = {{refresh.response.body.access_token}}
@ENDPOINT = https://{{OSDU_HOST}}
@LEGAL_HOST = {{ENDPOINT}}/api/legal/v1
@data_partition_id = opendes
@tag = opendes-public-usa-dataset-1
# -----------------------
# API: legal
# -----------------------
###
# @name getAllTag
GET {{LEGAL_HOST}}/legaltags
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
###
# @name create_tag
POST {{LEGAL_HOST}}/legaltags
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"name": "{{tag}}",
"description": "This tag is used by Data Upload Scripts",
"properties": {
"countryOfOrigin": [
"US"
],
"contractId": "A1234",
"expirationDate": "2020-12-31",
"originator": "MyCompany",
"dataType": "Transferred Data",
"securityClassification": "Public",
"personalData": "No Personal Data",
"exportClassification": "EAR99"
}
}
###
# @name getTag
GET {{LEGAL_HOST}}/legaltags/{{tag}}
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
###
# @name deleteTag
DELETE {{LEGAL_HOST}}/legaltags/{{tag}}
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
tools/rest/search.http 0 → 100644
View file @ a2c5b93d
This diff is collapsed.
tools/rest/storage.http 0 → 100644
View file @ a2c5b93d
# -------HTTP REST CLIENT -------
# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
## This script provides a few samples for calling storage.
# -----------------------
# OAUTH (Variables)
# -----------------------
###
@login_base = login.microsoftonline.com/{{TENANT_ID}}
@oauth_token_host = {{login_base}}/oauth2/v2.0/token
@scopes = {{CLIENT_ID}}/.default openid profile offline_access
# -----------------------
# OAUTH refresh_token
# -----------------------
###
# @name refresh
POST https://{{oauth_token_host}} HTTP/1.1
Content-Type: application/x-www-form-urlencoded
grant_type=refresh_token
&client_id={{CLIENT_ID}}
&client_secret={{CLIENT_SECRET}}
&refresh_token={{INITIAL_TOKEN}}
&scope={{scopes}}
# -----------------------
# API (Variables)
# -----------------------
###
@access_token = {{refresh.response.body.access_token}}
@ENDPOINT = https://{{OSDU_HOST}}
@STORAGE_HOST = {{ENDPOINT}}/api/storage/v2
@data_partition_id = opendes
@kind = opendes:qa-test:well-master:0.2.0
@id = <a_valid_document_id>
# -----------------------
# API: storage
# -----------------------
###
# @name getAllSchema
GET {{STORAGE_HOST}}/query/kinds
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
###
# @name getAllSchemaOfKind
GET {{STORAGE_HOST}}/query/kinds
Authorization: Bearer {{access_token}}
Accept: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "opendes:osdu:*:*",
"limit":1,
"offset":0,
"aggregateBy": "kind"
}
###
# @name create_schema
POST {{STORAGE_HOST}}/schemas
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"kind": "{{kind}}",
"schema": [
{
"path": "ResourceTypeID",
"kind": "string",
"ext": {}
},
{
"path": "ResourceID",
"kind": "string",
"ext": {}
},
{
"path": "ResourceHomeRegionID",
"kind": "string",
"ext": {}
},
{
"path": "ResourceHostRegionIDs",
"kind": "[]string",
"ext": {}
},
{
"path": "ResourceObjectCreationDateTime",
"kind": "string",
"ext": {}
},
{
"path": "ResourceVersionCreationDateTime",
"kind": "string",
"ext": {}
},
{
"path": "ResourceCurationStatus",
"kind": "string",
"ext": {}
},
{
"path": "ResourceLifecycleStatus",
"kind": "string",
"ext": {}
},
{
"path": "ResourceSecurityClassification",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.FacilityID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.FacilityTypeID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.FacilityOperator.FacilityOperatorOrganisationID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.FacilityOperator.EffectiveDateTime",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.FacilityOperator.TerminationDateTime",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.DataSourceOrganisationID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.SpatialLocation.SpatialLocationCoordinatesDate",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.SpatialLocation.QuantitativeAccuracyBandID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.SpatialLocation.QualitativeSpatialAccuracyTypeID",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.SpatialLocation.CoordinateQualityCheckPerformedBy",
"kind": "string",
"ext": {}
},
{
"path": "Data.IndividualTypeProperties.SpatialLocation.CoordinateQualityCheckDateTime",
"kind": "string",
"ext": {}
},
{