EDS Monitoring - Ingestion: Enhance Activity Template from OSDU ingest by reference in EDS Ingest

Problem

During an Osdu_ingest_by_reference, the following XCOM is available but not inside the Activity Template

Value

• Clear status report of the ingestion for osdu ingest by reference
• Able to articulate the activity logs (errors/success logs) in the Activity template
• Data Manager is able to resolve any errors easily from the activity template

Airflow XCOM screenshot

Activity Template

URL: POST https://osdu.core-dev.gcp.gnrg-osdu.projects.epam.com/api/search/v2/query_with_cursor

Body:

{
    "kind": "osdu:wks:work-product-component--Activity:*",
    "query": "id : \"osdu:work-product-component--Activity:eds_run_id_000d4c99-bec2-4aba-9a68-e8546fd1aab0\""
}

Response:

{
    "cursor": "7EFEF9DE1502A5370B23B8DF2F4F604D",
    "results": [
        {
            "data": {
                "SpatialArea.QuantitativeAccuracyBandID": null,
                "Description": null,
                "ParentActivityID": null,
                "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
                "ResourceLifecycleStatus": null,
                "SpatialArea.SpatialParameterTypeID": null,
                "ActivityTemplateID": null,
                "ResourceCurationStatus": null,
                "TechnicalAssuranceID": null,
                "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
                "LastActivityState.ActivityStatusID": null,
                "SpatialArea.SpatialGeometryTypeID": null,
                "Source": null,
                "IsExtendedLoad": true,
                "Name": null,
                "VirtualProperties.DefaultName": null,
                "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
                "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
                "ResourceSecurityClassification": null,
                "ExistenceKind": null,
                "LastActivityState.Remark": null,
                "Parameters": [
                    {
                        "StringParameter": "['osdu:reference-data--SampleType:medium-12mb-1', 'osdu:reference-data--SampleType:medium-12mb-2', 'osdu:reference-data--SampleType:medium-12mb-4', 'osdu:reference-data--SampleType:medium-12mb-3']",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "failed_records",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "[]",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "ingested_records",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "['71175ed9-5066-4eed-b04a-47086faa5fe9']",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "osdu_run_id",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "osdu:master-data--ConnectedSourceRegistryEntry:EDS-eds-m25-testing:",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "csre_id",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "{'71175ed9-5066-4eed-b04a-47086faa5fe9': '2025-03-19T01:15:03.923091Z'}",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "execution_start_date",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "{'71175ed9-5066-4eed-b04a-47086faa5fe9': '2025-03-19T01:15:23.833172Z'}",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "execution_end_date",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "0",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "total_ingested_records",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "4",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "total_failed_records",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "None",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "error_details",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "[]",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "reference_error",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    },
                    {
                        "StringParameter": "None",
                        "ParameterRoleID": "osdu:reference-data--ParameterRole:Input:",
                        "DataObjectParameter": null,
                        "Selection": null,
                        "Title": "post_ingestion_response",
                        "ParameterKindID": "osdu:reference-data--ParameterKind:String:",
                        "DataQuantityParameterUOMID": null
                    }
                ],
                "SpatialPoint.SpatialParameterTypeID": null,
                "SpatialPoint.QuantitativeAccuracyBandID": null,
                "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
                "SpatialArea.CoordinateQualityCheckPerformedBy": null,
                "SpatialPoint.SpatialGeometryTypeID": null,
                "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
                "ResourceHomeRegionID": null,
                "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
                "SubmitterName": null,
                "ParentProjectID": "osdu:master-data--ConnectedSourceDataJob:EDS-eds-m25-testing:",
                "SpatialPoint.QualitativeSpatialAccuracyTypeID": null
            },
            "kind": "osdu:wks:work-product-component--Activity:1.4.0",
            "source": "wks",
            "acl": {
                "viewers": [
                    "data.default.viewers@osdu.group"
                ],
                "owners": [
                    "data.default.owners@osdu.group"
                ]
            },
            "type": "work-product-component--Activity",
            "version": 1742346964420582,
            "tags": {
                "normalizedKind": "osdu:wks:work-product-component--Activity:1"
            },
            "createTime": "2025-03-19T01:16:04.428Z",
            "authority": "osdu",
            "namespace": "osdu:wks",
            "legal": {
                "legaltags": [
                    "osdu-eds-public-usa-dataset"
                ],
                "otherRelevantDataCountries": [
                    "US"
                ],
                "status": "compliant"
            },
            "createUser": "airflow@service.local",
            "id": "osdu:work-product-component--Activity:eds_run_id_000d4c99-bec2-4aba-9a68-e8546fd1aab0"
        }
    ],
    "totalCount": 1
}