M23 AZURE Manifest Ingestion Unit Conversion Without Persistable Reference Working Partially
Hello,
Testing the Manifest ingestion at Azure I noticed that the Frame of Reference - Unit conversion is working partially without the persistable reference. When the persistableReference is not included the API converts ms to s but not ft to m. When the persistableReference is included the API converts everything as expected.
You can find my documentation for Manifest Ingestion FoR Unit Conversion here.
Bellow you will find my work for Manifest Ingestion FoR Unit Conversion Without Persistabe Reference:
1. Create legal tag
POST https://{{LEGAL_HOST}}/legaltags
Curl
curl --location 'https://osdu-ship.msft-osdu-test.org/api/legal/v1/legaltags' \
--header 'Content-Type: application/json' \
--header 'data-partition-id: opendes' \
--header 'Authorization: Bearer <Token>' \
--data '{
"name": "opendes-R3FullManifest-Legal-Tag-Test9021234",
"description": "Legal Tag added for Well",
"properties": {
"contractId": "123456",
"countryOfOrigin": [
"US",
"CA"
],
"dataType": "Third Party Data",
"exportClassification": "EAR99",
"originator": "Schlumberger",
"personalData": "No Personal Data",
"securityClassification": "Private",
"expirationDate": "2025-12-25"
}
}'
Body
{
"name": "{{tagName}}",
"description": "Legal Tag added for Well",
"properties": {
"contractId": "123456",
"countryOfOrigin": [
"US",
"CA"
],
"dataType": "Third Party Data",
"exportClassification": "EAR99",
"originator": "Schlumberger",
"personalData": "No Personal Data",
"securityClassification": "Private",
"expirationDate": "2025-12-25"
}
}
Response
{
"name": "opendes-R3FullManifest-Legal-Tag-Test9021234",
"description": "Legal Tag added for Well",
"properties": {
"countryOfOrigin": [
"US",
"CA"
],
"contractId": "123456",
"expirationDate": "2025-12-25",
"originator": "Schlumberger",
"dataType": "Third Party Data",
"securityClassification": "Private",
"personalData": "No Personal Data",
"exportClassification": "EAR99"
}
}
2. WP Ingestion File Collection Without Persistable Reference (Seismic)
POST https://{{WORKFLOW_URL}}/workflow/Osdu_ingest/workflowRun
Curl
curl --location 'https://osdu-ship.msft-osdu-test.org/api/workflow/v1/workflow/Osdu_ingest/workflowRun' \
--header 'data-partition-id: opendes' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer <Token>' \
--header 'Cookie: JSESSIONID=E3AD6D7D9AEDA07FEF07C59894425A70' \
--data-raw '{
"executionContext": {
"Payload": {
"AppKey": "test-app",
"data-partition-id": "opendes"
},
"manifest": {
"kind": "opendes:wks:Manifest:1.0.0",
"ReferenceData": [
{
"id": "opendes:reference-data--ResourceSecurityClassification:RESTRICTED",
"kind": "osdu:wks:reference-data--ResourceSecurityClassification:1.0.0",
"data": {
"Source": "Volve Loading",
"Name": "RESTRICTED"
},
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-R3FullManifest-Legal-Tag-Test9021234"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
}
}
],
"MasterData": [
{
"id": "opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR",
"kind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"meta": [
{
"kind": "Unit",
"name": "ms",
"persistableReference": "",
"unitOfMeasureID": "opendes:reference-data--UnitOfMeasure:ms:",
"propertyNames": [
"RecordLength"
]
},
{
"kind": "Unit",
"name": "ft",
"persistableReference": "",
"unitOfMeasureID": "opendes:reference-data--UnitOfMeasure:ft:",
"propertyNames": [
"ShotpointIncrementDistance",
"CableLength",
"CableSpacingDistance",
"MaxOffsetDistance"
]
}
],
"data": {
"Purpose": "Acquisition for Volve",
"SeismicGeometryTypeID": "opendes:reference-data--SeismicGeometryType:3D:",
"OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
"ShotpointIncrementDistance": 25.0,
"EnergySourceTypeID": "opendes:reference-data--SeismicEnergySourceType:Airgun:",
"SourceArrayCount": 2,
"SourceArraySeparationDistance": 100.0,
"SampleInterval": 4.0,
"RecordLength": 10200,
"CableCount": 4,
"CableLength": 6000.0,
"CableSpacingDistance": 100.0,
"MaxOffsetDistance": 6000.0,
"FoldCount": 120,
"VesselNames": [
"GECO ANGLER"
]
},
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-R3FullManifest-Legal-Tag-Test9021234"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
}
}
]
}
}
}'
Body
{
"executionContext": {
"Payload": {
"AppKey": "test-app",
"data-partition-id": "{{data-partition-id}}"
},
"manifest": {
"kind": "{{data-partition-id}}:wks:Manifest:1.0.0",
"ReferenceData": [
{
"id": "{{data-partition-id}}:reference-data--ResourceSecurityClassification:RESTRICTED",
"kind": "{{authority}}:wks:reference-data--ResourceSecurityClassification:1.0.0",
"data": {
"Source": "Volve Loading",
"Name": "RESTRICTED"
},
"acl": {
"owners": [
"data.default.owners@{{data-partition-id}}{{domain}}"
],
"viewers": [
"data.default.viewers@{{data-partition-id}}{{domain}}"
]
},
"legal": {
"legaltags": [
"{{LegalTagNameExists}}"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
}
}
],
"MasterData": [
{
"id": "{{data-partition-id}}:master-data--SeismicAcquisitionSurvey:ST0202R08-{{randomId}}-FoR-Unit-JFA-WPR",
"kind": "{{authority}}:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"meta": [
{
"kind": "Unit",
"name": "ms",
"persistableReference": "",
"unitOfMeasureID": "{{data-partition-id}}:reference-data--UnitOfMeasure:ms:",
"propertyNames": [
"RecordLength"
]
},
{
"kind": "Unit",
"name": "ft",
"persistableReference": "",
"unitOfMeasureID": "{{data-partition-id}}:reference-data--UnitOfMeasure:ft:",
"propertyNames": [
"ShotpointIncrementDistance",
"CableLength",
"CableSpacingDistance",
"MaxOffsetDistance"
]
}
],
"data": {
"Purpose": "Acquisition for Volve",
"SeismicGeometryTypeID": "{{data-partition-id}}:reference-data--SeismicGeometryType:3D:",
"OperatingEnvironmentID": "{{data-partition-id}}:reference-data--OperatingEnvironment:Offshore:",
"ShotpointIncrementDistance": 25.0,
"EnergySourceTypeID": "{{data-partition-id}}:reference-data--SeismicEnergySourceType:Airgun:",
"SourceArrayCount": 2,
"SourceArraySeparationDistance": 100.0,
"SampleInterval": 4.0,
"RecordLength": 10200,
"CableCount": 4,
"CableLength": 6000.0,
"CableSpacingDistance": 100.0,
"MaxOffsetDistance": 6000.0,
"FoldCount": 120,
"VesselNames": [
"GECO ANGLER"
]
},
"acl": {
"owners": [
"data.default.owners@{{data-partition-id}}{{domain}}"
],
"viewers": [
"data.default.viewers@{{data-partition-id}}{{domain}}"
]
},
"legal": {
"legaltags": [
"{{LegalTagNameExists}}"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
}
}
]
}
}
}
Response
{
"workflowId": "Osdu_ingest",
"runId": "cea3b805-3838-413f-bf36-b0c5641f06d7",
"startTimeStamp": 1716429516522,
"status": "submitted",
"submittedBy": "3456711e-f381-410d-8b74-2345398d58d7"
}
3. WorkFlow - Get Work Status
GET https://{{WORKFLOW_URL}}/workflow/Osdu_ingest/workflowRun/{{run_id}}
Curl
curl --location 'https://osdu-ship.msft-osdu-test.org/api/workflow/v1/workflow/Osdu_ingest/workflowRun/cea3b805-3838-413f-bf36-b0c5641f06d7' \
--header 'Data-Partition-Id: opendes' \
--header 'Authorization: Bearer <Token>' \
--header 'Cookie: JSESSIONID=E3AD6D7D9AEDA07FEF07C59894425A70' \
--data ''
Response
{
"workflowId": "Osdu_ingest",
"runId": "cea3b805-3838-413f-bf36-b0c5641f06d7",
"startTimeStamp": 1716429516522,
"endTimeStamp": 1716429543678,
"status": "finished",
"submittedBy": "3456711e-f381-410d-8b74-2345398d58d7"
}
4. XCom Summary - Process_single_manifest_file_task
- Access Airflow: https://osdu-ship.msft-osdu-test.org/airflow2/home
- Access DAG “Osdu_ingest”
- Change the view from “Tree” to “Graph”
- Check the runID the you got on postman
- Click on the graph on “Process_single_manifest_file_task”
- A window will pop up, click on “log” on the top
- Change from “log” to “XCom"
record_ids ['opendes:reference-data--ResourceSecurityClassification:RESTRICTED', 'opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR']
5. Retrieve the Master Data Record Using Storage
Curl
curl --location 'https://osdu-ship.msft-osdu-test.org/api/storage/v2/records/opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR' \
--header 'Data-Partition-Id: opendes' \
--header 'Authorization: Bearer <Token>' \
--data ''
Response
{
"data": {
"CableCount": 4,
"CableLength": 6000,
"CableSpacingDistance": 100,
"EnergySourceTypeID": "opendes:reference-data--SeismicEnergySourceType:Airgun:",
"FoldCount": 120,
"MaxOffsetDistance": 6000,
"OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
"Purpose": "Acquisition for Volve",
"RecordLength": 10200,
"SampleInterval": 4,
"SeismicGeometryTypeID": "opendes:reference-data--SeismicGeometryType:3D:",
"ShotpointIncrementDistance": 25,
"SourceArrayCount": 2,
"SourceArraySeparationDistance": 100,
"VesselNames": [
"GECO ANGLER"
]
},
"meta": [
{
"kind": "Unit",
"name": "ms",
"persistableReference": "",
"propertyNames": [
"RecordLength"
],
"unitOfMeasureID": "opendes:reference-data--UnitOfMeasure:ms:"
},
{
"kind": "Unit",
"name": "ft",
"persistableReference": "",
"propertyNames": [
"ShotpointIncrementDistance",
"CableLength",
"CableSpacingDistance",
"MaxOffsetDistance"
],
"unitOfMeasureID": "opendes:reference-data--UnitOfMeasure:ft:"
}
],
"id": "opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR",
"version": 1716429539127475,
"kind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"acl": {
"viewers": [
"data.default.viewers@opendes.contoso.com"
],
"owners": [
"data.default.owners@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-R3FullManifest-Legal-Tag-Test9021234"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
},
"createUser": "3456711e-f381-410d-8b74-2345398d58d7",
"createTime": "2024-05-23T01:58:59.726Z"
}
6. Retrieve the Master Data Using Search
POST https://{{SEARCH_HOST}}/query
Curl
curl --location 'https://osdu-ship.msft-osdu-test.org/api/search/v2/query' \
--header 'Content-Type: application/json' \
--header 'data-partition-id: opendes' \
--header 'Authorization: Bearer <Token>' \
--data '{
"kind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"query" : "id:\"opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR\""
}'
Body
{
"kind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"query" : "id:\"{{data-partition-id}}:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR\""
}
Response
{
"results": [
{
"data": {
"Operator": null,
"SourceArrayCount": 2,
"ProjectName": null,
"ShotpointIncrementDistance": 25.0,
"VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
"ResourceLifecycleStatus": null,
"ProjectID": null,
"CableCount": 4,
"CableLength": 6000.0,
"ResourceCurationStatus": null,
"TechnicalAssuranceID": null,
"VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
"Source": null,
"SourceArraySeparationDistance": 100.0,
"VirtualProperties.DefaultName": null,
"CableSpacingDistance": 100.0,
"VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
"VersionCreationReason": null,
"ResourceSecurityClassification": null,
"SpatialLocation.SpatialParameterTypeID": null,
"ExistenceKind": null,
"SpatialLocation.CoordinateQualityCheckPerformedBy": null,
"VesselNames": [
"GECO ANGLER"
],
"MaxOffsetDistance": 6000.0,
"TechnicalAssuranceTypeID": null,
"Purpose": "Acquisition for Volve",
"RecordLength": 10.200000000000001,
"FoldCount": 120,
"SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
"SpatialLocation.SpatialGeometryTypeID": null,
"OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
"VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
"ResourceHomeRegionID": null,
"SeismicGeometryTypeID": "opendes:reference-data--SeismicGeometryType:3D:",
"VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
"SampleInterval": 4.0,
"EnergySourceTypeID": "opendes:reference-data--SeismicEnergySourceType:Airgun:",
"SpatialLocation.QuantitativeAccuracyBandID": null
},
"kind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
"source": "wks",
"acl": {
"viewers": [
"data.default.viewers@opendes.contoso.com"
],
"owners": [
"data.default.owners@opendes.contoso.com"
]
},
"type": "master-data--SeismicAcquisitionSurvey",
"version": 1716429539127475,
"tags": {
"normalizedKind": "osdu:wks:master-data--SeismicAcquisitionSurvey:1"
},
"createTime": "2024-05-23T01:58:59.726Z",
"authority": "osdu",
"namespace": "osdu:wks",
"legal": {
"legaltags": [
"opendes-R3FullManifest-Legal-Tag-Test9021234"
],
"otherRelevantDataCountries": [
"US"
],
"status": "compliant"
},
"createUser": "3456711e-f381-410d-8b74-2345398d58d7",
"id": "opendes:master-data--SeismicAcquisitionSurvey:ST0202R08-566430-FoR-Unit-JFA-WPR"
}
],
"aggregations": null,
"phraseSuggestions": [],
"totalCount": 1
}
Conversion Check
RecordLength
10200 ms = 10.200000000000001 s
ShotpointIncrementDistance - DIDN'T WORK
25 ft = 25 ft X
CableLength - DIDN'T WORK
6000 ft = 6000 ft X
CableSpacingDistance - DIDN'T WORK
100 ft = 100 ft X
MaxOffsetDistance - DIDN'T WORK
6000 ft = 6000 ft X