Commit b5d5eb66 authored by Chad Leong's avatar Chad Leong
Browse files

initial repo

parent 55098640
LAS Loader utility for Wellbore DDMS
# LAS Loader utility for Wellbore DDMS
## Description
1. This will be the OSDU LAS loader utility to support Wellbore DDMS.
2. This is a wrapper behind the Wellbore DDMS [API](https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services/-/blob/master/spec/generated/openapi.json)
{
"info": {
"_postman_id": "a10239e0-e3b7-44bc-8749-86ae82409036",
"name": "Azure Preship",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
{
"name": "Get Bearer Token - Wellbore DDMS",
"event": [
{
"listen": "test",
"script": {
"exec": [
"// this snippet extracts the new access and id tokens\r",
"// and puts them to environment variables\r",
"\r",
"var data = JSON.parse(responseBody)\r",
"pm.environment.set(\"access_token\", data.access_token);\r",
"pm.environment.set(\"refresh_token\", data.refresh_token);"
],
"type": "text/javascript"
}
}
],
"request": {
"method": "POST",
"header": [],
"body": {
"mode": "urlencoded",
"urlencoded": [
{
"key": "grant_type",
"value": "refresh_token",
"type": "text"
},
{
"key": "client_id",
"value": "ab320ed3-9cdd-4798-8e3c-2a657800183b",
"type": "text"
},
{
"key": "client_secret",
"value": "aBLGVsZaXsI83bSeicGQKqP2~YD_vf-XUR",
"type": "text"
},
{
"key": "scope",
"value": "ab320ed3-9cdd-4798-8e3c-2a657800183b/.default openid profile offline_access",
"type": "text"
},
{
"key": "refresh_token",
"value": "0.ATcA01-XWHdJ0ES-qDevC6rBANMOMqvdnJhHjjwqZXgAGDs3AMU.AgABAAAAAAD--DLA3VO7QrddgJg7WevrAgDs_wQA9P8Iv1xNr9laoueeoxis1kIpRGOOEOgp0NprM9u_OUTGQPWlImcBbNRKJOH367iIzEYbRAACtXv_9P9gh5vYasAk5kn6GT0wnHlfHyEn3sCINU8JblkD_At76rnr4Q46yHLj1LXRbum4ILgLm6L8t-xjkIWmaBCsO3oYJfHkosPMroAGDheoo4BHS25HbU_HCqUpIMWP7e_JMyVXXYnUCjNtGIoLdwrtLfvW7d4i4B15FZdjv3ACcV-hrpzxGzdv5F8F0VQuNpdVM1BMBvU09npoS5eC8YLyRvzEiJuO1JJZKfCWzmnJT7EF_V_xEgwavHDO6Ux_ATbL-6dFL_QuCOpJ_lIjmGiS1JR-9PvLU1fjt-aN86SKUzN6Jn4wKnZfuuz1imXJ24QXYR0Fhc7D7DRXfYe0O4N45m35ICmhD2sh5ngro_xtQ329SEEEWDjMdk8g3Naop0WLlpnAPrtlND4mAB6l_GFGZzKR4_ThwuGHN6dLvGjNvDVsvEWr3ltLdUaWFcdUfXUwBUzZRqyTW9YfQMrw25uUB8_NCsaJEfoxOaCZX9iASQX_lIZL-DuMrNgCfuDACajZOibcJPkmzhOKtpCxl-zWzEHYyQY3WM64zw8DCBhPwNee-q98qvBFxqUcS-FMZPcYBSyt3ElJ2cnnto9DT-KN6CxB_r3fx3MzrVkJB3Pm3Zw26o99v0Iu4mp47HkEFH__uimjn7pjbK2sOxZ5UpOjXm88h8UvtCknViRIQPhPR1V94_Uk755BA4JDjwyd4aDO5YVCTSYkst1S5EurxlBa6iWphIpLJLPe-VqjZYPH4v0iC1MVXDRlcUlCTKvpV1Ew7mAx4aKi9MC4E6kc8zqAQHXYzgOKPPLVirCK8Xtl3jlxxLPCb185F6PFpdGUXdi9kbmeXXfl0IRhUS-_SNDV_wYaf-4bx1TkguFkPHLtOx6aKPDtpg31pMz2o_jFnWvEqPlETseI27iPDohj_Ygx7Z-iYy_XRQ-uVsQBmBLG6chbyzvS5bK1VxkzK40v08UribbYkfmJdO2NE25bXWU",
"type": "text"
}
]
},
"url": {
"raw": "https://login.microsoftonline.com/58975fd3-4977-44d0-bea8-37af0baac100/oauth2/v2.0/token",
"protocol": "https",
"host": [
"login",
"microsoftonline",
"com"
],
"path": [
"58975fd3-4977-44d0-bea8-37af0baac100",
"oauth2",
"v2.0",
"token"
]
}
},
"response": []
}
]
}
\ No newline at end of file
This diff is collapsed.
import lasio
import httpx
import io
import json
import pandas as pd
import urllib
###############################
# OSDU CSP Access Information #
###############################
# CSP Auth
BaseUrl = "https://osdu-ship.msft-osdu-test.org" # Azure
access_token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Im5PbzNaRHJPRFhFSzFqS1doWHNsSFJfS1hFZyIsImtpZCI6Im5PbzNaRHJPRFhFSzFqS1doWHNsSFJfS1hFZyJ9.eyJhdWQiOiJhYjMyMGVkMy05Y2RkLTQ3OTgtOGUzYy0yYTY1NzgwMDE4M2IiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC81ODk3NWZkMy00OTc3LTQ0ZDAtYmVhOC0zN2FmMGJhYWMxMDAvIiwiaWF0IjoxNjMwNjYxNzM5LCJuYmYiOjE2MzA2NjE3MzksImV4cCI6MTYzMDY2NTYzOSwiYWNyIjoiMSIsImFpbyI6IkFUUUF5LzhUQUFBQW9uQ2o4VFBCOXFXTk1xbGhWQUNadEYra2JoWUxuWEhoNURRMWUrZGkvejVERThydjZVSk9idmtYL2VyKzVoNG4iLCJhbXIiOlsicHdkIl0sImFwcGlkIjoiYWIzMjBlZDMtOWNkZC00Nzk4LThlM2MtMmE2NTc4MDAxODNiIiwiYXBwaWRhY3IiOiIxIiwiZmFtaWx5X25hbWUiOiJEZWZhdWx0IiwiZ2l2ZW5fbmFtZSI6IlByZXNoaXAiLCJpcGFkZHIiOiIxODMuODMuNDUuMzAiLCJuYW1lIjoiUHJlc2hpcHBpbmciLCJvaWQiOiI4ZTBiNDY0NC0wZWQwLTQxZjItOGM2ZC1kNmFhYTc5ODYxODIiLCJyaCI6IjAuQVRjQTAxLVhXSGRKMEVTLXFEZXZDNnJCQU5NT01xdmRuSmhIamp3cVpYZ0FHRHMzQU1VLiIsInNjcCI6IlVzZXIuUmVhZCIsInN1YiI6Ijk3cFFnSnRSRkg5OVkxS1Zpd0ZWNEdhQUR4S3NJZVJHOVpQSi00UG5NYjAiLCJ0aWQiOiI1ODk3NWZkMy00OTc3LTQ0ZDAtYmVhOC0zN2FmMGJhYWMxMDAiLCJ1bmlxdWVfbmFtZSI6InByZXNoaXBwaW5nQGF6dXJlZ2xvYmFsMS5vbm1pY3Jvc29mdC5jb20iLCJ1cG4iOiJwcmVzaGlwcGluZ0BhenVyZWdsb2JhbDEub25taWNyb3NvZnQuY29tIiwidXRpIjoiY2w3SzhWVTl2RUNOQlBxcWRFdk9BQSIsInZlciI6IjEuMCJ9.UTtN50qaFgLqucozbnCscEiqFZ2UHRiXJJYUuyv2QqTWX0zSObeOMf1NH-ipnF1d5WuhZIjbge6e9dPX-najeCbNj46m2Io9kTjePejErkaY19H9BC36cq1ZddVoknisHBVFVU7I426_oLdcyZvcvs-Cbj0dvSST_1Ypa-qdTu_RartHu1c9bT36wXKD6NEH7vhf2yse2SsacNnh5zaIU6aoKAE0QorWUxb7dAOILnSQ16tMhTPxrj96yUu22UAS2Qtldr1g2E0aDMK0e_wbay46vzGoWfkG-WXfoNhg-cBZUYp-d79MQiHri1_oEMmZNpAsGg-Y25n1BvsBJlXYJQ"
data_partition_id = "opendes"
acl_domain = "contoso.com"
legal_tag = "opendes-public-usa-dataset-7643990"
acl = {
"viewers": [f"data.default.viewers@{data_partition_id}.{acl_domain}"],
"owners": [f"data.default.owners@{data_partition_id}.{acl_domain}"],
}
legal = {"legaltags": [legal_tag], "otherRelevantDataCountries": ["US"], "status": "compliant"}
# Authorization header
headers = {
"Authorization": f"Bearer {access_token}",
"data-partition-id": data_partition_id,
}
####################################
# Parsing LAS File for header info #
####################################
print("(1/6) Parsing Las File for header info...\n")
# LAS File name input
log_filename = "15_9-19_SR_CPI.las"
# Read LAS with lasio
las = lasio.read(log_filename)
# Get Well Name
well_name = las.well.WELL.value
# Get Unique Well Identifier (UWI)
uwi = las.well.UWI.value
########################################
# Create wellbore record from LAS data #
########################################
print("(2/6) Creating wellbore record from LAS data...\n")
# Create Wellbore Record from Well Name and UWI
wellbore_record = {
"acl": acl,
"data": {
"FacilityName": well_name,
"NameAliases": [
{
"AliasName": uwi,
"AliasNameTypeID": f"{data_partition_id}:reference-data--AliasNameType:UniqueIdentifier:",
}
],
},
"kind": "osdu:wks:master-data--Wellbore:1.0.0",
"legal": legal,
}
# Writing wellbore record to OSDU
payload = [wellbore_record]
post_record_url = f"{BaseUrl}/api/os-wellbore-ddms/ddms/v3/wellbores"
res = httpx.post(post_record_url, json=payload, headers=headers)
# Saving wellbore id from Resposne
wellbore_id = res.json()["recordIds"][0]
print(f"Wellbore record created successfully. Wellbore record ID: {wellbore_id}\n")
########################################
# Create well log record from LAS data #
########################################
print("(3/6) Creating well log record from LAS data...\n")
# Building curves object
curves = []
for curve in las.curves:
unit = urllib.parse.quote(curve.unit, safe="").replace(" ", "-")
if unit == "":
unit = "UNITLESS"
curves.append(
{
"CurveID": curve.mnemonic,
"CurveUnit": f"opendes:reference-data--UnitOfMeasure:{unit}:",
"Mnemonic": curve.mnemonic,
}
)
# Creating well log record from curves object
welllog_record = {
"acl": acl,
"data": {
"ReferenceCurveID": "DEPT",
"Curves": curves,
"WellboreID": f"{wellbore_id}:",
},
"kind": "osdu:wks:work-product-component--WellLog:1.1.0",
"legal": legal,
}
# Write well log record to OSDU and saving welllog_id
payload = [welllog_record]
post_welllogs_url = f"{BaseUrl}/api/os-wellbore-ddms/ddms/v3/welllogs"
res = httpx.post(post_welllogs_url, json=payload, headers=headers)
welllog_id = res.json()["recordIds"][0]
print(f"Well log record created succesfully. Well Log record ID: {welllog_id}\n")
######################################
# Writing bulk data to Wellbore DDMS #
######################################
print("(4/6) Writing bulk data to Wellbore DDMS...\n")
# Resetting headers in PandasDataFrame
df = las.df().reset_index()
# Chunking
welllog_data_url = f"{BaseUrl}/api/os-wellbore-ddms/alpha/ddms/v3/welllogs/{welllog_id}/data"
# Writing Bulk data to URL
parquet_headers = {"content-type": "application/x-parquet"}
parquet_headers.update(headers)
df.to_parquet()
res = httpx.post(welllog_data_url, content=df.to_parquet(), headers=parquet_headers)
if res.status_code == 200:
print("Bulk data written succesfully to wellbore DDMS data store. \n")
else:
print("Error, please check data source")
###########################################################
# Recognize log family and add property to ingested data #
###########################################################
print("(5/6) Recognizing log family and adding property to ingested data...\n")
def fetch_wellbore_record(headers, record_id):
get_record_url = f"{BaseUrl}/api/os-wellbore-ddms/ddms/v3/wellbores/{record_id}"
res = httpx.get(get_record_url, headers=headers)
return res.json()
def fetch_welllog_record(headers, record_id):
get_record_url = f"{BaseUrl}/api/os-wellbore-ddms/ddms/v3/welllogs/{record_id}"
res = httpx.get(get_record_url, headers=headers)
return res.json()
# Fetching ingested wellog record with welllog id
welllog_record = fetch_welllog_record(headers, welllog_id)
# Recognizing log family
recognize_family_url = f"{BaseUrl}/api/os-wellbore-ddms/log-recognition/family"
curves = welllog_record["data"]["Curves"]
for curve in curves:
mnemonic = curve["Mnemonic"]
unit = las.curves[mnemonic].unit
payload = {"label": mnemonic, "log_unit": unit}
res = httpx.post(recognize_family_url, json=payload, headers=headers)
try:
print("Recognizing log family curve: " + mnemonic)
family_id = urllib.parse.quote(res.json()["family"], safe="").replace(" ", "-")
curve["LogCurveFamilyID"] = f"{data_partition_id}:reference-data--LogCurveFamily:{family_id}:"
except:
pass
#########################################################
# Updating well log record with new enriched log family #
#########################################################
print("Updating new log property with enriched log family...\n")
# # Modyfing new record
payload = [welllog_record]
post_welllogs_url = f"{BaseUrl}/api/os-wellbore-ddms/ddms/v3/welllogs"
res = httpx.post(post_welllogs_url, json=payload, headers=headers)
record_id = res.json()["recordIds"][0]
print(f"Updated Well Log Record ID: {record_id} \n")
#############################################
# Retrieving Well Log data in Wellbore DDMS #
#############################################
print("(6/6) Retrieving ingested Well Log Data in Wellbore DDMs...\n")
# Retreive well log data
def describe_welllog_data(headers, record_id):
describe_welllog_data_url = f"{BaseUrl}/api/os-wellbore-ddms/alpha/ddms/v3/welllogs/{record_id}/data?describe=true"
res = httpx.get(describe_welllog_data_url, headers=headers)
return res.json()
welllog_description = describe_welllog_data(headers, welllog_id)
print(f"Well log record {welllog_id} - Header Info: ")
print(json.dumps(welllog_description, sort_keys=True))
print("\n")
def fetch_welllog_data(headers, record_id, curves):
curves_query = ""
if curves:
curves_query = "&curves={}".format(",".join(curves))
get_welllog_data_url = (
f"{BaseUrl}/api/os-wellbore-ddms/alpha/ddms/v3/welllogs/{record_id}/data?describe=false{curves_query}"
)
res = httpx.get(get_welllog_data_url, headers=headers)
assert res.status_code == 200, res.content
welllog_bytes = io.BytesIO(res.content)
df = pd.read_parquet(welllog_bytes)
return df
print(f"Well log record {welllog_id} - Log Data: ")
df = fetch_welllog_data(headers, record_id, welllog_description["columns"])
print(df)
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment