Commit 229d8618 authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

GONRG-2726: Python SDK libs and providers updates

parent ccb43296
# https://editorconfig.org/
root = true
[*]
indent_style = space
indent_size = 4
insert_final_newline = true
trim_trailing_whitespace = true
end_of_line = lf
charset = utf-8
[*.py]
max_line_length = 100
......@@ -6,4 +6,5 @@ __pycache__
**/venv/**
**/.idea/**
.vscode/
.DS_Store
\ No newline at end of file
.DS_Store
.pytest_cache
......@@ -7,6 +7,8 @@
* [Getting Started](#getting-started)
* [Testing](#testing)
* * [Running E2E Tests](#running-e2e-tests)
* * [Running Ingestion libs Tests](#running-ingestion-libs-tests)
* * [Running CSP Tests](#running-csp-tests)
* [Licence](#licence)
......@@ -17,12 +19,19 @@ Interactions with OSDU services are cloud platform-agnostic by design. However,
platforms, and the OSDU R3 Prototype provides a dedicated Python SDK to make sure that interactions are independent from the
cloud platforms.
The Python SDK must be installed on the machine that uses OSDU services. Each cloud provider needs to
modify and configure this SDK to run on their cloud platform.
The Python SDK must be installed on the machine that uses OSDU services.
In OSDU R3 Prototype, the SDK encapsulates calls to the ODES Storage and Search services. In the future releases, the
SDK might provide additional interfaces, in particular, for the Ingestion service.
In OSDU R3 Prototype, the SDK encapsulates calls to the ODES Storage and Search services.
The SDK now provides different components for the ingestion process in `osdu_api.libs` folder. Among them:
- validating OSDU entities against corresponding schemas;
- ensuring referential integrity;
- finding parent-child relationships between entities;
- etc.
Also, in `osdu_api.providers` folder the SDK provides common interfaces for writing cloud-specific implementations for authorization and accessing
cloud storages. In this `osdu_api.providers` folder CSP code is stored.
# Getting Started
......@@ -33,19 +42,19 @@ To install this package:
2. Use Python 3.6. Also, it is highly recommended using an isolated virtual environment for development purposes
(Creation of virtual environments: https://docs.python.org/3.6/library/venv.html)
3. Make sure you have setuptools and wheel installed
```sh
3. Make sure you have setuptools and wheel installed
```sh
pip install --upgrade setuptools wheel
```
4. Change directory to the root of PythonSDK project
```sh
4. Change directory to the root of PythonSDK project
```sh
cd path/to/python-sdk
```
5. Make sure osdu-api isn't already installed
5. Make sure osdu-api isn't already installed
```sh
pip uninstall osdu-api
````
......@@ -57,7 +66,7 @@ pip install -r requirements.txt
```
7. Install Python SDK
```sh
python setup.py install
```
......@@ -74,6 +83,24 @@ Specify of end-services URLs into `tests/osdu_api.yaml` and run
pytest test
```
### Running ingestion libs tests
```shell
export CLOUD_PROVIDER=provider_test
pip install -r requirements.txt
pip install -r requirements-dev.txt
pytest ./osdu_api/test/libs-unit-tests
```
### Running CSP tests
```shell
export CLOUD_PROVIDER=<cloud_provider>
pip install -r requirements.txt
pip install -r requirements-dev.txt
pip install -r ./osdu_api/test/providers-unit-tests/<cloud_provider>/requirements-test.txt
pytest ./osdu_api/test/libs-unit-tests
```
## Licence
Copyright © Amazon Web Services
......
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys, os
import importlib
from configparser import SafeConfigParser
import requests
from osdu_api.model.http_method import HttpMethod
'''
Base client that is meant to be extended by service specific clients
'''
class BaseClient:
'''
Base client gets initialized with configuration values and a bearer token
based on provider-specific logic
'''
def __init__(self):
self._parse_config()
self.unauth_retries = 0
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self._refresh_service_principal_token()
'''
Example config file:
[environment]
data_partition_id=opendes
storage_url=https://[STORAGE_ENDPOINT]/api/storage/v2
search_url=https://[SEARCH_ENDPOINT]/api/search/v2
data_workflow_url=https://[WORKFLOW_ENDPOINT]/api/data-workflow/v1
file_dms_url=https://[FILE_DMS_ENDPOINT]/api/filedms/v2
dataset_registry_url=https://[DATASET_REGISTRY_URL]/api/dataset-registry/v1
[provider]
name=aws
entitlements_module_name=entitlements_client
'''
def _parse_config(self):
config_parser = SafeConfigParser(os.environ)
config_file_name = 'osdu_api.ini'
found_names = config_parser.read(config_file_name)
if config_file_name not in found_names:
raise Exception('Could not find osdu_api.ini config file')
self.data_partition_id = config_parser.get('environment', 'data_partition_id')
self.storage_url = config_parser.get('environment', 'storage_url')
self.search_url = config_parser.get('environment', 'search_url')
self.data_workflow_url = config_parser.get('environment', 'data_workflow_url')
self.file_dms_url = config_parser.get('environment', 'file_dms_url')
self.legal_url = config_parser.get('environment', 'legal_url')
self.entitlements_url = config_parser.get('environment', 'entitlements_url')
self.dataset_url = config_parser.get('environment', 'dataset_url')
self.use_service_principal = config_parser.get('environment', 'use_service_principal')
self.provider = config_parser.get('provider', 'name')
self.service_principal_module_name = config_parser.get('provider', 'service_principal_module_name')
'''
The path to the logic to get a valid bearer token is dynamically injected based on
what provider and entitlements module name is provided in the configuration yaml
'''
def _refresh_service_principal_token(self):
entitlements_client = importlib.import_module('osdu_api.providers.%s.%s' % (self.provider, self.service_principal_module_name))
self.service_principal_token = entitlements_client.get_service_principal_token()
'''
Makes a request using python's built in requests library. Takes additional headers if
necessary
'''
def make_request(self, method: HttpMethod, url: str, data = '', add_headers = {}, params = {}, bearer_token = None):
if bearer_token is not None and 'Bearer ' not in bearer_token:
bearer_token = 'Bearer ' + bearer_token
headers = {
'content-type': 'application/json',
'data-partition-id': self.data_partition_id,
'Authorization': bearer_token if bearer_token is not None else self.service_principal_token
}
if (len(add_headers) > 0):
for key, value in add_headers:
headers[key] = value
response = object
if (method == HttpMethod.GET):
response = requests.get(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.DELETE):
response = requests.delete(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.POST):
response = requests.post(url=url, params=params, data=data, headers=headers, verify=False)
elif (method == HttpMethod.PUT):
response = requests.put(url=url, params=params, data=data, headers=headers, verify=False)
if (response.status_code == 401 or response.status_code == 403) and self.unauth_retries < 1:
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self.unauth_retries += 1
self._refresh_service_principal_token()
self.make_request(method, url, data, add_headers, params, None)
self.unauth_retries = 0
return response
......@@ -11,10 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys, os
import importlib
import os
from configparser import SafeConfigParser
import requests
from osdu_api.model.http_method import HttpMethod
......@@ -32,7 +34,7 @@ class BaseClient:
self.unauth_retries = 0
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self._refresh_service_principal_token()
def _parse_config(self):
"""
Example config file:
......@@ -74,7 +76,7 @@ class BaseClient:
"""
entitlements_client = importlib.import_module('osdu_api.provider.%s.%s' % (self.provider, self.service_principal_module_name))
self.service_principal_token = entitlements_client.get_service_principal_token()
def make_request(self, method: HttpMethod, url: str, data = '', add_headers = {}, params = {}, bearer_token = None):
"""
Makes a request using python's built in requests library. Takes additional headers if
......@@ -82,7 +84,7 @@ class BaseClient:
"""
if bearer_token is not None and 'Bearer ' not in bearer_token:
bearer_token = 'Bearer ' + bearer_token
headers = {
'content-type': 'application/json',
'data-partition-id': self.data_partition_id,
......@@ -109,7 +111,7 @@ class BaseClient:
self.unauth_retries += 1
self._refresh_service_principal_token()
self.make_request(method, url, data, add_headers, params, None)
self.unauth_retries = 0
return response
......@@ -14,11 +14,13 @@
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.start_workflow import StartWorkflow
from osdu_api.model.data_workflow.update_status_request import UpdateStatusRequest
from osdu_api.model.http_method import HttpMethod
class DataWorkflowClient(BaseClient):
"""
Holds the logic for interfacing with Data Workflow's api
......
......@@ -14,9 +14,10 @@
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.data_workflow.get_workflow_schedules_request import GetWorkflowSchedulesRequest
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.http_method import HttpMethod
......
......@@ -13,6 +13,7 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
from osdu_api.model.http_method import HttpMethod
......
......@@ -13,6 +13,7 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.create_dataset_registries_request import CreateDatasetRegistriesRequest
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
......
......@@ -13,10 +13,12 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.entitlements.group import Group
from osdu_api.model.entitlements.group_member import GroupMember
from osdu_api.model.http_method import HttpMethod
class EntitlementsClient(BaseClient):
"""
......
......@@ -13,11 +13,12 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.legal.legal_tag import LegalTag
from osdu_api.model.legal.update_legal_tag import UpdateLegalTag
from osdu_api.model.legal.legal_tag_names import LegalTagNames
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.legal.update_legal_tag import UpdateLegalTag
class LegalClient(BaseClient):
......
......@@ -13,8 +13,10 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
#from osdu_api.model.storage.schema.schema import Schema
#from osdu_api.model.storage.schema.schema_attribute import SchemaAttribute
......
......@@ -13,10 +13,11 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.search.query_response import QueryResponse
from osdu_api.model.search.query_request import QueryRequest
from osdu_api.model.search.query_response import QueryResponse
class SearchClient(BaseClient):
......
......@@ -13,10 +13,11 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.storage.record import Record
from osdu_api.model.storage.query_records_request import QueryRecordsRequest
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.storage.query_records_request import QueryRecordsRequest
from osdu_api.model.storage.record import Record
class RecordClient(BaseClient):
......
......@@ -13,11 +13,13 @@
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.storage.schema.schema import Schema
from osdu_api.model.storage.schema.schema_attribute import SchemaAttribute
class SchemaClient(BaseClient):
"""
Holds the logic for interfacing with Storage's R2 schema api
......
......@@ -17,13 +17,14 @@ Basic example on creating a record and searching on it to get it back
import os
import time
from osdu_api.clients.storage.record_client import RecordClient
from osdu_api.clients.search.search_client import SearchClient
from osdu_api.model.storage.record import Record
from osdu_api.clients.storage.record_client import RecordClient
from osdu_api.model.search.query_request import QueryRequest
from osdu_api.model.storage.acl import Acl
from osdu_api.model.storage.legal import Legal
from osdu_api.model.storage.legal import Legal
from osdu_api.model.storage.record import Record
from osdu_api.model.storage.record_ancestry import RecordAncestry
from osdu_api.model.search.query_request import QueryRequest
record_client = RecordClient()
search_client = SearchClient()
......
......@@ -14,18 +14,19 @@
import json
from osdu_api.clients.data_workflow.data_workflow_client import DataWorkflowClient
from osdu_api.clients.data_workflow.data_workflow_scheduling_client import DataWorkflowSchedulingClient
from osdu_api.clients.data_workflow.data_workflow_scheduling_client import \
DataWorkflowSchedulingClient
from osdu_api.clients.dataset_registry.dataset_registry_client import DatasetRegistryClient
from osdu_api.clients.file_dms.file_dms_client import FileDMSClient
from osdu_api.model.data_workflow.get_workflow_schedules_request import GetWorkflowSchedulesRequest
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.dataset_registry.create_dataset_registries import CreateDatasetRegistries
from osdu_api.model.file_dms.file import File
from osdu_api.model.file_dms.register_files import RegisterFiles
from osdu_api.model.storage.record import Record
from osdu_api.model.storage.acl import Acl
from osdu_api.model.storage.legal import Legal
from osdu_api.model.storage.record import Record
from osdu_api.model.storage.record_ancestry import RecordAncestry
from osdu_api.model.dataset_registry.create_dataset_registries import CreateDatasetRegistries
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.data_workflow.get_workflow_schedules_request import GetWorkflowSchedulesRequest
data_workflow_client = DataWorkflowClient()
data_workflow_scheduling_client = DataWorkflowSchedulingClient()
......
......@@ -14,12 +14,13 @@
# limitations under the License.
import logging
from typing import Callable, Union
from abc import ABC, abstractmethod
from functools import partial
from http import HTTPStatus
from typing import Callable, Union
import requests
from osdu_api.libs.exceptions import TokenRefresherNotPresentError
logger = logging.getLogger()
......
......@@ -17,6 +17,7 @@ import enum
import os
import yaml
from osdu_api.libs.exceptions import ConfigurationError
"""
......
......@@ -26,7 +26,7 @@ DATA_TYPES_WITH_SURROGATE_KEYS = ("dataset", "work-product", "work-product-compo
SURROGATE_KEYS_PATHS = [
("definitions", "{{data-partition-id}}:wks:AbstractWPCGroupType:1.0.0", "properties", "Datasets",
"items"),
("definitions", "osdu:wks:AbstractWPCGroupType:1.0.0", "properties", "Artefacts",
("definitions", "{{data-partition-id}}:wks:AbstractWPCGroupType:1.0.0", "properties", "Artefacts",
"items", "properties", "ResourceID"),
("properties", "data", "allOf", 1, "properties", "Components", "items"),
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment