AWS - 50,000 Performance Loading Fails
#50000 Records fails - below is the error from the python shell console.
"kind": "osdu:wks:master-data--Organisation:1.0.0", "acl": {"viewers": ["data.default.viewers@osdu.example.com"], "owners": ["data.default.owners@osdu.example.com"]}, "legal": {"legaltags": ["osdu-public-usa-dataset"], "otherRelevantDataCountries": ["US"]}, "data": {"OrganisationName": "Name_TEST_BULK_2022-03-01_174_49999", "Source": "Source_TEST_BULK_2022-03-01_174_49999"}}, {"id": "osdu:master-data--Organisation:TEST_BULK_2022-03-01_174_50000", "kind": "osdu:wks:master-data--Organisation:1.0.0", "acl": {"viewers": ["data.default.viewers@osdu.example.com"], "owners": ["data.default.owners@osdu.example.com"]}, "legal": {"legaltags": ["osdu-public-usa-dataset"], "otherRelevantDataCountries": ["US"]}, "data": {"OrganisationName": "Name_TEST_BULK_2022-03-01_174_50000", "Source": "Source_TEST_BULK_2022-03-01_174_50000"}}]}}}, "headers": {"data-partition-id": "osdu", "Accept": "application/json", "Authorization": "Bearer {hidded}"}, "verify": false} DEBUG:root:Response: 401 DEBUG:root:json = {"status": 401, "message": "The user is not authorized to perform this action", "reason": "Access denied"}
Please note, #1,000 records passed ok; and was able to retrieve the records.
aws_client.get_record('osdu:master-data--Organisation:TEST_BULK_2022-03-01_233_999').dump() DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): r3m10.preshiptesting.osdu.aws:443 DEBUG:urllib3.connectionpool:https://r3m10.preshiptesting.osdu.aws:443 "GET /api/storage/v2/records/osdu:master-data--Organisation:TEST_BULK_2022-03-01_233_999 HTTP/1.1" 200 567 DEBUG:root:HTTP GET https://r3m10.preshiptesting.osdu.aws/api/storage/v2/records/osdu:master-data--Organisation:TEST_BULK_2022-03-01_233_999 DEBUG:root:args = {"headers": {"data-partition-id": "osdu", "Accept": "application/json", "Authorization": "Bearer {Hidden}"}, "verify": false} DEBUG:root:Response: 200 DEBUG:root:json = {"data": {"OrganisationName": "Name_TEST_BULK_2022-03-01_233_999", "Source": "Source_TEST_BULK_2022-03-01_233_999"}, "meta": [], "id": "osdu:master-data--Organisation:TEST_BULK_2022-03-01_233_999", "version": 1646147532729232, "kind": "osdu:wks:master-data--Organisation:1.0.0", "acl": {"viewers": ["data.default.viewers@osdu.example.com"], "owners": ["data.default.owners@osdu.example.com"]}, "legal": {"legaltags": ["osdu-public-usa-dataset"], "otherRelevantDataCountries": ["US"], "status": "compliant"}, "createUser": "serviceprincipal@testing.com", "createTime": "2022-03-01T15:12:12.736Z"} { "data": { "OrganisationName": "Name_TEST_BULK_2022-03-01_233_999", "Source": "Source_TEST_BULK_2022-03-01_233_999" }, "meta": [], "id": "osdu:master-data--Organisation:TEST_BULK_2022-03-01_233_999", "version": 1646147532729232, "kind": "osdu:wks:master-data--Organisation:1.0.0", "acl": { "viewers": [ "data.default.viewers@osdu.example.com" ], "owners": [ "data.default.owners@osdu.example.com" ] }, "legal": { "legaltags": [ "osdu-public-usa-dataset" ], "otherRelevantDataCountries": [ "US" ], "status": "compliant" }, "createUser": "serviceprincipal@testing.com", "createTime": "2022-03-01T15:12:12.736Z" }