Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
SDKs
Python SDK
Commits
af3b0499
Commit
af3b0499
authored
Oct 21, 2021
by
Spencer Sutton
Browse files
Merge branch 'aws-update' into 'master'
Adding partition service client See merge request
!48
parents
d775a68c
8f1fe5a5
Pipeline
#73407
passed with stages
in 55 seconds
Changes
7
Pipelines
3
Hide whitespace changes
Inline
Side-by-side
osdu_api/clients/base_client.py
View file @
af3b0499
...
...
@@ -14,7 +14,7 @@
import
importlib
import
os
from
configparser
import
SafeConfigParser
import
logging
import
os
import
requests
...
...
@@ -28,7 +28,7 @@ class BaseClient:
Base client that is meant to be extended by service specific clients
"""
def
__init__
(
self
,
config_manager
:
BaseConfigManager
=
None
,
data_partition_id
=
None
):
def
__init__
(
self
,
config_manager
:
BaseConfigManager
=
None
,
data_partition_id
=
None
,
logger
=
None
):
"""
Base client gets initialized with configuration values and a bearer token
based on provider-specific logic
...
...
@@ -37,6 +37,10 @@ class BaseClient:
self
.
unauth_retries
=
0
if
self
.
use_service_principal
:
self
.
_refresh_service_principal_token
()
self
.
logger
=
logger
if
self
.
logger
is
None
:
self
.
logger
=
logging
.
getLogger
(
__name__
)
def
_parse_config
(
self
,
config_manager
:
BaseConfigManager
=
None
,
data_partition_id
=
None
):
"""
...
...
@@ -45,27 +49,28 @@ class BaseClient:
:param config_manager: ConfigManager to get configs, defaults to None
:type config_manager: BaseConfigManager, optional
"""
config_manager
=
config_manager
or
DefaultConfigManager
()
self
.
provider
=
config_manager
.
get
(
'provider'
,
'name'
)
self
.
data_workflow_url
=
config_manager
.
get
(
'environment'
,
'data_workflow_url'
)
self
.
dataset_url
=
config_manager
.
get
(
'environment'
,
'dataset_url'
)
self
.
entitlements_url
=
config_manager
.
get
(
'environment'
,
'entitlements_url'
)
self
.
file_dms_url
=
config_manager
.
get
(
'environment'
,
'file_dms_url'
)
self
.
legal_url
=
config_manager
.
get
(
'environment'
,
'legal_url'
)
self
.
schema_url
=
config_manager
.
get
(
'environment'
,
'schema_url'
)
self
.
search_url
=
config_manager
.
get
(
'environment'
,
'search_url'
)
self
.
storage_url
=
config_manager
.
get
(
'environment'
,
'storage_url'
)
self
.
ingestion_workflow_url
=
config_manager
.
get
(
'environment'
,
'ingestion_workflow_url'
)
self
.
provider
=
config_manager
.
get
(
'provider'
,
'name'
)
self
.
use_service_principal
=
config_manager
.
getbool
(
'environment'
,
'use_service_principal'
,
False
)
self
.
config_manager
=
config_manager
or
DefaultConfigManager
()
self
.
provider
=
self
.
config_manager
.
get
(
'provider'
,
'name'
)
self
.
data_workflow_url
=
self
.
config_manager
.
get
(
'environment'
,
'data_workflow_url'
)
self
.
dataset_url
=
self
.
config_manager
.
get
(
'environment'
,
'dataset_url'
)
self
.
entitlements_url
=
self
.
config_manager
.
get
(
'environment'
,
'entitlements_url'
)
self
.
file_dms_url
=
self
.
config_manager
.
get
(
'environment'
,
'file_dms_url'
)
self
.
legal_url
=
self
.
config_manager
.
get
(
'environment'
,
'legal_url'
)
self
.
schema_url
=
self
.
config_manager
.
get
(
'environment'
,
'schema_url'
)
self
.
search_url
=
self
.
config_manager
.
get
(
'environment'
,
'search_url'
)
self
.
storage_url
=
self
.
config_manager
.
get
(
'environment'
,
'storage_url'
)
self
.
partition_url
=
self
.
config_manager
.
get
(
'environment'
,
'partition_url'
)
self
.
ingestion_workflow_url
=
self
.
config_manager
.
get
(
'environment'
,
'ingestion_workflow_url'
)
self
.
provider
=
self
.
config_manager
.
get
(
'provider'
,
'name'
)
self
.
use_service_principal
=
self
.
config_manager
.
getbool
(
'environment'
,
'use_service_principal'
,
False
)
if
self
.
use_service_principal
:
self
.
service_principal_module_name
=
config_manager
.
get
(
'provider'
,
'service_principal_module_name'
)
self
.
service_principal_module_name
=
self
.
config_manager
.
get
(
'provider'
,
'service_principal_module_name'
)
if
data_partition_id
is
None
:
self
.
data_partition_id
=
config_manager
.
get
(
'environment'
,
'data_partition_id'
)
self
.
data_partition_id
=
self
.
config_manager
.
get
(
'environment'
,
'data_partition_id'
)
else
:
self
.
data_partition_id
=
data_partition_id
...
...
osdu_api/clients/partition/__init__.py
0 → 100644
View file @
af3b0499
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
osdu_api/clients/partition/partition_client.py
0 → 100644
View file @
af3b0499
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
os
import
logging
import
json
import
importlib
import
requests
from
osdu_api.clients.base_client
import
BaseClient
from
osdu_api.providers.aws.service_principal_util
import
get_service_principal_token
from
osdu_api.model.http_method
import
HttpMethod
class
PartitionClient
(
BaseClient
):
"""Mirrors the pattern laid out in os core common for the java services by
getting a service principal token and passing that to partition service. It
then will pass along the response to Cloud Service Provider (CSP) specific code
for a transformation into the final response.
"""
def
get_partition
(
self
,
data_partition_id
:
str
,
bearer_token
:
str
=
None
)
->
dict
:
"""[summary]
Args:
data_partition_id (str): standard OSDU data partition id (osdu, opendes, etc.)
bearer_token (str, optional): will be used instead of service principal token
Raises:
Exception: only when data partition id arg is empty
err: only when response from partition service is bad
Returns:
dict: CSP-specific partition info
"""
if
data_partition_id
is
None
:
raise
Exception
(
"data partition id cannot be empty"
)
if
bearer_token
is
None
:
# explicitly passing service principal token because this use case
# requires it be used even when other use cases have it disabled
self
.
_refresh_service_principal_token
()
bearer_token
=
self
.
service_principal_token
self
.
logger
.
info
(
"Successfully retrieved token"
)
partition_info_converter_module_name
=
self
.
config_manager
.
get
(
'provider'
,
'partition_info_converter_module'
)
self
.
logger
.
debug
(
f
"Partition converter module name:
{
partition_info_converter_module_name
}
"
)
partition_info_converter
=
importlib
.
import_module
(
'osdu_api.providers.%s.%s'
%
(
self
.
provider
,
partition_info_converter_module_name
))
csp_response
=
{}
try
:
response
=
self
.
make_request
(
method
=
HttpMethod
.
GET
,
url
=
'{}{}/{}'
.
format
(
self
.
partition_url
,
'/partitions'
,
data_partition_id
),
bearer_token
=
bearer_token
)
response
.
raise_for_status
()
content
=
json
.
loads
(
response
.
content
)
csp_response
=
partition_info_converter
.
convert
(
content
)
except
requests
.
exceptions
.
HTTPError
as
err
:
self
.
logger
.
error
(
f
"Received status code
{
response
.
status_code
}
from partition service"
)
raise
err
return
csp_response
osdu_api/providers/aws/partition_converter_aws.py
0 → 100644
View file @
af3b0499
from
osdu_api.providers.aws.partition_info_aws
import
PartitionInfoAws
# Example response from partition service:
# {
# "tenantSSMPrefix": {
# "sensitive": false,
# "value": "/osdu/<prefix>/shared"
# },
# "policy-service-enabled": {
# "sensitive": false,
# "value": "false"
# },
# "tenantId": {
# "sensitive": false,
# "value": "shared"
# },
# "resourcePrefix": {
# "sensitive": false,
# "value": "<prefix>"
# },
# "id": {
# "sensitive": false,
# "value": "osdu"
# }
# }
def
convert
(
content
:
dict
)
->
PartitionInfoAws
:
"""Convert response from partition service to python object
Args:
content (dict): json response from partition service
Returns:
PartitionInfoAws: has attributes defining partition info
"""
return
PartitionInfoAws
(
content
[
"id"
][
"value"
],
content
[
"tenantId"
][
"value"
],
content
[
"resourcePrefix"
][
"value"
],
content
[
"tenantSSMPrefix"
][
"value"
]
)
\ No newline at end of file
osdu_api/providers/aws/partition_info_aws.py
0 → 100644
View file @
af3b0499
class
PartitionInfoAws
():
def
__init__
(
self
,
id
:
str
,
tenant_id
:
str
,
resource_prefix
:
str
,
tenant_ssm_prefix
:
str
):
self
.
id
=
id
self
.
tenant_id
=
tenant_id
self
.
resource_prefix
=
resource_prefix
self
.
tenant_ssm_prefix
=
tenant_ssm_prefix
osdu_api/test/osdu_api.ini
View file @
af3b0499
...
...
@@ -22,11 +22,13 @@ file_dms_url=blah/api/filedms/v2
dataset_url
=
blah/api/dataset-registry/v1
schema_url
=
blah/api/schema-service/v1
ingestion_workflow_url
=
stub
partition_url
=
stub
use_service_principal
=
True
[provider]
name
=
aws
service_principal_module_name
=
service_principal_util
partition_info_converter_module
=
partition_converter_aws
token_url_ssm_path
=
/osdu/blah/oauth-token-uri
aws_oauth_custom_scope_ssm_path
=
/osdu/blah/oauth-custom-scope
client_id_ssm_path
=
/osdu/blah/client-credentials-client-id
...
...
setup.py
View file @
af3b0499
...
...
@@ -50,6 +50,13 @@ setuptools.setup(
"License :: OSI Approved :: Apache License"
,
"Operating System :: OS Independent"
,
],
install_requires
=
[
"jsonschema==3.2.0"
,
"pyyaml==5.4.1"
,
"strict-rfc3339==0.7"
,
"toposort==1.6"
,
"dataclasses==0.8;python_version<'3.7'"
],
extras_require
=
{
"all"
:
[
"requests==2.25.1"
,
"tenacity==6.2.0"
]
},
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment