Merge branch 'aws-update' into 'master'

Adding partition service client

See merge request !48

osdu_api/clients/base_client.py

@@ -14,7 +14,7 @@
 import importlib
 import os
 from configparser import SafeConfigParser
-
+import logging
 import os
 import requests
 
@@ -28,7 +28,7 @@ class BaseClient:
     Base client that is meant to be extended by service specific clients
     """
 
-    def __init__(self, config_manager: BaseConfigManager = None, data_partition_id = None):
+    def __init__(self, config_manager: BaseConfigManager = None, data_partition_id = None, logger = None):
         """
         Base client gets initialized with configuration values and a bearer token
         based on provider-specific logic
@@ -37,6 +37,10 @@ class BaseClient:
         self.unauth_retries = 0
         if self.use_service_principal:
             self._refresh_service_principal_token()
+        
+        self.logger = logger
+        if self.logger is None:
+            self.logger = logging.getLogger(__name__)
 
     def _parse_config(self, config_manager: BaseConfigManager = None, data_partition_id = None):
         """
@@ -45,27 +49,28 @@ class BaseClient:
         :param config_manager: ConfigManager to get configs, defaults to None
         :type config_manager: BaseConfigManager, optional
         """
-        config_manager = config_manager or DefaultConfigManager()
-
-        self.provider = config_manager.get('provider', 'name')
-
-        self.data_workflow_url = config_manager.get('environment', 'data_workflow_url')
-        self.dataset_url = config_manager.get('environment', 'dataset_url')
-        self.entitlements_url = config_manager.get('environment', 'entitlements_url')
-        self.file_dms_url = config_manager.get('environment', 'file_dms_url')
-        self.legal_url = config_manager.get('environment', 'legal_url')
-        self.schema_url = config_manager.get('environment', 'schema_url')
-        self.search_url = config_manager.get('environment', 'search_url')
-        self.storage_url = config_manager.get('environment', 'storage_url')
-        self.ingestion_workflow_url = config_manager.get('environment', 'ingestion_workflow_url')
-        self.provider = config_manager.get('provider', 'name')
-
-        self.use_service_principal = config_manager.getbool('environment', 'use_service_principal', False)
+        self.config_manager = config_manager or DefaultConfigManager()
+
+        self.provider = self.config_manager.get('provider', 'name')
+
+        self.data_workflow_url = self.config_manager.get('environment', 'data_workflow_url')
+        self.dataset_url = self.config_manager.get('environment', 'dataset_url')
+        self.entitlements_url = self.config_manager.get('environment', 'entitlements_url')
+        self.file_dms_url = self.config_manager.get('environment', 'file_dms_url')
+        self.legal_url = self.config_manager.get('environment', 'legal_url')
+        self.schema_url = self.config_manager.get('environment', 'schema_url')
+        self.search_url = self.config_manager.get('environment', 'search_url')
+        self.storage_url = self.config_manager.get('environment', 'storage_url')
+        self.partition_url = self.config_manager.get('environment', 'partition_url')
+        self.ingestion_workflow_url = self.config_manager.get('environment', 'ingestion_workflow_url')
+        self.provider = self.config_manager.get('provider', 'name')
+
+        self.use_service_principal = self.config_manager.getbool('environment', 'use_service_principal', False)
         if self.use_service_principal:
-            self.service_principal_module_name = config_manager.get('provider', 'service_principal_module_name')
+            self.service_principal_module_name = self.config_manager.get('provider', 'service_principal_module_name')
 
         if data_partition_id is None:
-            self.data_partition_id = config_manager.get('environment', 'data_partition_id')
+            self.data_partition_id = self.config_manager.get('environment', 'data_partition_id')
         else:
             self.data_partition_id = data_partition_id
 

osdu_api/clients/partition/__init__.py

+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

osdu_api/clients/partition/partition_client.py

+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import json
+import importlib
+import requests
+
+from osdu_api.clients.base_client import BaseClient
+from osdu_api.providers.aws.service_principal_util import get_service_principal_token
+from osdu_api.model.http_method import HttpMethod
+
+class PartitionClient(BaseClient):
+    """Mirrors the pattern laid out in os core common for the java services by
+    getting a service principal token and passing that to partition service. It 
+    then will pass along the response to Cloud Service Provider (CSP) specific code
+    for a transformation into the final response.
+    """
+        
+    def get_partition(self, data_partition_id: str, bearer_token:str = None) -> dict:
+        """[summary]
+
+        Args:
+            data_partition_id (str): standard OSDU data partition id (osdu, opendes, etc.)
+            bearer_token (str, optional): will be used instead of service principal token
+
+        Raises:
+            Exception: only when data partition id arg is empty
+            err: only when response from partition service is bad
+
+        Returns:
+            dict: CSP-specific partition info
+        """
+
+        if data_partition_id is None:
+            raise Exception("data partition id cannot be empty")
+        
+        if bearer_token is None:
+            # explicitly passing service principal token because this use case 
+            # requires it be used even when other use cases have it disabled
+            self._refresh_service_principal_token()
+            bearer_token = self.service_principal_token
+            self.logger.info("Successfully retrieved token")
+
+        partition_info_converter_module_name = self.config_manager.get('provider', 'partition_info_converter_module')
+        self.logger.debug(f"Partition converter module name: {partition_info_converter_module_name}")
+
+        partition_info_converter = importlib.import_module('osdu_api.providers.%s.%s' % (self.provider, partition_info_converter_module_name))
+
+        csp_response = {}
+        try:
+            response = self.make_request(method=HttpMethod.GET, url='{}{}/{}'.format(self.partition_url, '/partitions', data_partition_id), 
+                bearer_token=bearer_token)
+            response.raise_for_status()
+
+            content = json.loads(response.content)
+
+            csp_response = partition_info_converter.convert(content)
+
+        except requests.exceptions.HTTPError as err:
+            self.logger.error(f"Received status code {response.status_code} from partition service")
+            raise err
+        
+        return csp_response
+
+        

osdu_api/providers/aws/partition_converter_aws.py

+
+from osdu_api.providers.aws.partition_info_aws import PartitionInfoAws
+
+# Example response from partition service:
+# {
+#     "tenantSSMPrefix": {
+#         "sensitive": false,
+#         "value": "/osdu/<prefix>/shared"
+#     },
+#     "policy-service-enabled": {
+#         "sensitive": false,
+#         "value": "false"
+#     },
+#     "tenantId": {
+#         "sensitive": false,
+#         "value": "shared"
+#     },
+#     "resourcePrefix": {
+#         "sensitive": false,
+#         "value": "<prefix>"
+#     },
+#     "id": {
+#         "sensitive": false,
+#         "value": "osdu"
+#     }
+# }
+
+def convert(content: dict) -> PartitionInfoAws:
+    """Convert response from partition service to python object
+
+    Args:
+        content (dict): json response from partition service
+
+    Returns:
+        PartitionInfoAws: has attributes defining partition info
+    """
+    return PartitionInfoAws(
+        content["id"]["value"],
+        content["tenantId"]["value"],
+        content["resourcePrefix"]["value"],
+        content["tenantSSMPrefix"]["value"]
+    )
+    
\ No newline at end of file

osdu_api/providers/aws/partition_info_aws.py

+class PartitionInfoAws():
+
+    def __init__(self, id: str, tenant_id: str, resource_prefix: str, tenant_ssm_prefix: str):
+        self.id = id
+        self.tenant_id = tenant_id
+        self.resource_prefix = resource_prefix
+        self.tenant_ssm_prefix = tenant_ssm_prefix

osdu_api/test/osdu_api.ini

@@ -22,11 +22,13 @@ file_dms_url=blah/api/filedms/v2
 dataset_url=blah/api/dataset-registry/v1
 schema_url=blah/api/schema-service/v1
 ingestion_workflow_url=stub
+partition_url=stub
 use_service_principal=True
 
 [provider]
 name=aws
 service_principal_module_name=service_principal_util
+partition_info_converter_module=partition_converter_aws
 token_url_ssm_path=/osdu/blah/oauth-token-uri
 aws_oauth_custom_scope_ssm_path=/osdu/blah/oauth-custom-scope
 client_id_ssm_path=/osdu/blah/client-credentials-client-id

setup.py

@@ -50,6 +50,13 @@ setuptools.setup(
         "License :: OSI Approved :: Apache License",
         "Operating System :: OS Independent",
     ],
+    install_requires=[
+        "jsonschema==3.2.0",
+        "pyyaml==5.4.1",
+        "strict-rfc3339==0.7",
+        "toposort==1.6",
+        "dataclasses==0.8;python_version<'3.7'"
+    ],
     extras_require={
         "all": ["requests==2.25.1", "tenacity==6.2.0"]
     },