Commit af3b0499 authored by Spencer Sutton's avatar Spencer Sutton
Browse files

Merge branch 'aws-update' into 'master'

Adding partition service client

See merge request !48
parents d775a68c 8f1fe5a5
Pipeline #73407 passed with stages
in 55 seconds
......@@ -14,7 +14,7 @@
import importlib
import os
from configparser import SafeConfigParser
import logging
import os
import requests
......@@ -28,7 +28,7 @@ class BaseClient:
Base client that is meant to be extended by service specific clients
"""
def __init__(self, config_manager: BaseConfigManager = None, data_partition_id = None):
def __init__(self, config_manager: BaseConfigManager = None, data_partition_id = None, logger = None):
"""
Base client gets initialized with configuration values and a bearer token
based on provider-specific logic
......@@ -37,6 +37,10 @@ class BaseClient:
self.unauth_retries = 0
if self.use_service_principal:
self._refresh_service_principal_token()
self.logger = logger
if self.logger is None:
self.logger = logging.getLogger(__name__)
def _parse_config(self, config_manager: BaseConfigManager = None, data_partition_id = None):
"""
......@@ -45,27 +49,28 @@ class BaseClient:
:param config_manager: ConfigManager to get configs, defaults to None
:type config_manager: BaseConfigManager, optional
"""
config_manager = config_manager or DefaultConfigManager()
self.provider = config_manager.get('provider', 'name')
self.data_workflow_url = config_manager.get('environment', 'data_workflow_url')
self.dataset_url = config_manager.get('environment', 'dataset_url')
self.entitlements_url = config_manager.get('environment', 'entitlements_url')
self.file_dms_url = config_manager.get('environment', 'file_dms_url')
self.legal_url = config_manager.get('environment', 'legal_url')
self.schema_url = config_manager.get('environment', 'schema_url')
self.search_url = config_manager.get('environment', 'search_url')
self.storage_url = config_manager.get('environment', 'storage_url')
self.ingestion_workflow_url = config_manager.get('environment', 'ingestion_workflow_url')
self.provider = config_manager.get('provider', 'name')
self.use_service_principal = config_manager.getbool('environment', 'use_service_principal', False)
self.config_manager = config_manager or DefaultConfigManager()
self.provider = self.config_manager.get('provider', 'name')
self.data_workflow_url = self.config_manager.get('environment', 'data_workflow_url')
self.dataset_url = self.config_manager.get('environment', 'dataset_url')
self.entitlements_url = self.config_manager.get('environment', 'entitlements_url')
self.file_dms_url = self.config_manager.get('environment', 'file_dms_url')
self.legal_url = self.config_manager.get('environment', 'legal_url')
self.schema_url = self.config_manager.get('environment', 'schema_url')
self.search_url = self.config_manager.get('environment', 'search_url')
self.storage_url = self.config_manager.get('environment', 'storage_url')
self.partition_url = self.config_manager.get('environment', 'partition_url')
self.ingestion_workflow_url = self.config_manager.get('environment', 'ingestion_workflow_url')
self.provider = self.config_manager.get('provider', 'name')
self.use_service_principal = self.config_manager.getbool('environment', 'use_service_principal', False)
if self.use_service_principal:
self.service_principal_module_name = config_manager.get('provider', 'service_principal_module_name')
self.service_principal_module_name = self.config_manager.get('provider', 'service_principal_module_name')
if data_partition_id is None:
self.data_partition_id = config_manager.get('environment', 'data_partition_id')
self.data_partition_id = self.config_manager.get('environment', 'data_partition_id')
else:
self.data_partition_id = data_partition_id
......
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import logging
import json
import importlib
import requests
from osdu_api.clients.base_client import BaseClient
from osdu_api.providers.aws.service_principal_util import get_service_principal_token
from osdu_api.model.http_method import HttpMethod
class PartitionClient(BaseClient):
"""Mirrors the pattern laid out in os core common for the java services by
getting a service principal token and passing that to partition service. It
then will pass along the response to Cloud Service Provider (CSP) specific code
for a transformation into the final response.
"""
def get_partition(self, data_partition_id: str, bearer_token:str = None) -> dict:
"""[summary]
Args:
data_partition_id (str): standard OSDU data partition id (osdu, opendes, etc.)
bearer_token (str, optional): will be used instead of service principal token
Raises:
Exception: only when data partition id arg is empty
err: only when response from partition service is bad
Returns:
dict: CSP-specific partition info
"""
if data_partition_id is None:
raise Exception("data partition id cannot be empty")
if bearer_token is None:
# explicitly passing service principal token because this use case
# requires it be used even when other use cases have it disabled
self._refresh_service_principal_token()
bearer_token = self.service_principal_token
self.logger.info("Successfully retrieved token")
partition_info_converter_module_name = self.config_manager.get('provider', 'partition_info_converter_module')
self.logger.debug(f"Partition converter module name: {partition_info_converter_module_name}")
partition_info_converter = importlib.import_module('osdu_api.providers.%s.%s' % (self.provider, partition_info_converter_module_name))
csp_response = {}
try:
response = self.make_request(method=HttpMethod.GET, url='{}{}/{}'.format(self.partition_url, '/partitions', data_partition_id),
bearer_token=bearer_token)
response.raise_for_status()
content = json.loads(response.content)
csp_response = partition_info_converter.convert(content)
except requests.exceptions.HTTPError as err:
self.logger.error(f"Received status code {response.status_code} from partition service")
raise err
return csp_response
from osdu_api.providers.aws.partition_info_aws import PartitionInfoAws
# Example response from partition service:
# {
# "tenantSSMPrefix": {
# "sensitive": false,
# "value": "/osdu/<prefix>/shared"
# },
# "policy-service-enabled": {
# "sensitive": false,
# "value": "false"
# },
# "tenantId": {
# "sensitive": false,
# "value": "shared"
# },
# "resourcePrefix": {
# "sensitive": false,
# "value": "<prefix>"
# },
# "id": {
# "sensitive": false,
# "value": "osdu"
# }
# }
def convert(content: dict) -> PartitionInfoAws:
"""Convert response from partition service to python object
Args:
content (dict): json response from partition service
Returns:
PartitionInfoAws: has attributes defining partition info
"""
return PartitionInfoAws(
content["id"]["value"],
content["tenantId"]["value"],
content["resourcePrefix"]["value"],
content["tenantSSMPrefix"]["value"]
)
\ No newline at end of file
class PartitionInfoAws():
def __init__(self, id: str, tenant_id: str, resource_prefix: str, tenant_ssm_prefix: str):
self.id = id
self.tenant_id = tenant_id
self.resource_prefix = resource_prefix
self.tenant_ssm_prefix = tenant_ssm_prefix
......@@ -22,11 +22,13 @@ file_dms_url=blah/api/filedms/v2
dataset_url=blah/api/dataset-registry/v1
schema_url=blah/api/schema-service/v1
ingestion_workflow_url=stub
partition_url=stub
use_service_principal=True
[provider]
name=aws
service_principal_module_name=service_principal_util
partition_info_converter_module=partition_converter_aws
token_url_ssm_path=/osdu/blah/oauth-token-uri
aws_oauth_custom_scope_ssm_path=/osdu/blah/oauth-custom-scope
client_id_ssm_path=/osdu/blah/client-credentials-client-id
......
......@@ -50,6 +50,13 @@ setuptools.setup(
"License :: OSI Approved :: Apache License",
"Operating System :: OS Independent",
],
install_requires=[
"jsonschema==3.2.0",
"pyyaml==5.4.1",
"strict-rfc3339==0.7",
"toposort==1.6",
"dataclasses==0.8;python_version<'3.7'"
],
extras_require={
"all": ["requests==2.25.1", "tenacity==6.2.0"]
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment