Commit cc82feef authored by Spencer Sutton's avatar Spencer Sutton
Browse files

Merge branch 'update' into 'master'

Update to have more endpoints (More storage, entitlements, dataset, file dms, etc.)

See merge request !3
parents a467c9e7 843a5c1f
......@@ -5,4 +5,4 @@ __pycache__
**/venv/**
**/.idea/**
.vscode/
\ No newline at end of file
.vscode/
......@@ -187,7 +187,7 @@
identification within third-party archives.
Copyright 2020 Google LLC
Copyright 2020 Amazon
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
......
# OSDU Python SDK
To install locally:
- Make sure you have setuptools and wheel installed `python3 -m pip install --user --upgrade setuptools wheel`
- Run `python setup.py sdist bdist_wheel`
- Make sure osdu-api isn't already installed `pip uninstall osdu-api`
- Run `python -m pip install <YOUR PATH TO PACKAGE>/dist/osdu_api-0.0.1-py3-none-any.whl` make sure to substitute your machine's path in that command
Example import after installing:
`from osdu_api.storage.record_client import RecordClient`
## Contents
......@@ -72,6 +80,7 @@ pytest test
## Licence
Copyright © Amazon Web Services
Copyright © Google LLC
Copyright © EPAM Systems
......@@ -85,4 +94,6 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
\ No newline at end of file
limitations under the License.
A package to interface with OSDU microservices
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
env:
variables:
# these are needed because unit tests check config parsing and config injects env vars
STORAGE_BASE_URL: stub
SEARCH_BASE_URL: stub
LEGAL_BASE_URL: stub
DATA_WORKFLOW_BASE_URL: stub
FILE_DMS_BASE_URL: stub
DATASET_REGISTRY_BASE_URL: stub
AUTH_TOKEN_URL: stub
CUSTOM_SCOPE: stub
ENVIRONMENT: stub
AWS_REGION: stub
phases:
install:
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
build:
commands:
# install packages
- pip install -r requirements.txt
# run basic build
- python setup.py build
# run unit tests
- cp osdu_api/test/osdu_api.ini osdu_api.ini
- python -m unittest discover osdu_api/test
- rm osdu_api.ini
# publish new artifact to code artifact
- aws codeartifact login --tool twine --domain osdu-dev --domain-owner 888733619319 --repository osdu-python
- python setup.py sdist bdist_wheel
- twine upload --skip-existing --repository codeartifact dist/osdu_api-0.0.4.tar.gz
artifacts:
files:
- "**/*"
# base-directory: ${OUTPUT_DIR}
name: test
\ No newline at end of file
# Copyright 2020 Google LLC
# Copyright 2020 Amazon
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......
# Copyright 2020 Google LLC
# Copyright 2020 Amazon
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys, os
import importlib
from configparser import SafeConfigParser
import requests
from osdu_api.model.http_method import HttpMethod
class BaseClient:
"""
Base client that is meant to be extended by service specific clients
"""
def __init__(self):
"""
Base client gets initialized with configuration values and a bearer token
based on provider-specific logic
"""
self._parse_config()
self.unauth_retries = 0
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self._refresh_service_principal_token()
def _parse_config(self):
"""
Example config file:
[environment]
data_partition_id=opendes
storage_url=https://[STORAGE_ENDPOINT]/api/storage/v2
search_url=https://[SEARCH_ENDPOINT]/api/search/v2
data_workflow_url=https://[WORKFLOW_ENDPOINT]/api/data-workflow/v1
file_dms_url=https://[FILE_DMS_ENDPOINT]/api/filedms/v2
dataset_registry_url=https://[DATASET_REGISTRY_URL]/api/dataset-registry/v1
[provider]
name=aws
entitlements_module_name=entitlements_client
"""
config_parser = SafeConfigParser(os.environ)
config_file_name = 'osdu_api.ini'
found_names = config_parser.read(config_file_name)
if config_file_name not in found_names:
raise Exception('Could not find osdu_api.ini config file')
self.data_partition_id = config_parser.get('environment', 'data_partition_id')
self.storage_url = config_parser.get('environment', 'storage_url')
self.search_url = config_parser.get('environment', 'search_url')
self.data_workflow_url = config_parser.get('environment', 'data_workflow_url')
self.file_dms_url = config_parser.get('environment', 'file_dms_url')
self.legal_url = config_parser.get('environment', 'legal_url')
self.entitlements_url = config_parser.get('environment', 'entitlements_url')
self.dataset_url = config_parser.get('environment', 'dataset_url')
self.use_service_principal = config_parser.get('environment', 'use_service_principal')
self.provider = config_parser.get('provider', 'name')
self.service_principal_module_name = config_parser.get('provider', 'service_principal_module_name')
def _refresh_service_principal_token(self):
"""
The path to the logic to get a valid bearer token is dynamically injected based on
what provider and entitlements module name is provided in the configuration yaml
"""
entitlements_client = importlib.import_module('osdu_api.provider.%s.%s' % (self.provider, self.service_principal_module_name))
self.service_principal_token = entitlements_client.get_service_principal_token()
def make_request(self, method: HttpMethod, url: str, data = '', add_headers = {}, params = {}, bearer_token = None):
"""
Makes a request using python's built in requests library. Takes additional headers if
necessary
"""
if bearer_token is not None and 'Bearer ' not in bearer_token:
bearer_token = 'Bearer ' + bearer_token
headers = {
'content-type': 'application/json',
'data-partition-id': self.data_partition_id,
'Authorization': bearer_token if bearer_token is not None else self.service_principal_token
}
if (len(add_headers) > 0):
for key, value in add_headers:
headers[key] = value
response = object
if (method == HttpMethod.GET):
response = requests.get(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.DELETE):
response = requests.delete(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.POST):
response = requests.post(url=url, params=params, data=data, headers=headers, verify=False)
elif (method == HttpMethod.PUT):
response = requests.put(url=url, params=params, data=data, headers=headers, verify=False)
if (response.status_code == 401 or response.status_code == 403) and self.unauth_retries < 1:
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self.unauth_retries += 1
self._refresh_service_principal_token()
self.make_request(method, url, data, add_headers, params, None)
self.unauth_retries = 0
return response
\ No newline at end of file
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
\ No newline at end of file
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.start_workflow import StartWorkflow
from osdu_api.model.data_workflow.update_status_request import UpdateStatusRequest
from osdu_api.model.http_method import HttpMethod
class DataWorkflowClient(BaseClient):
"""
Holds the logic for interfacing with Data Workflow's api
"""
def start_workflow(self, start_workflow: StartWorkflow, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/startWorkflow'),
data=start_workflow.to_JSON(), bearer_token=bearer_token)
def update_status(self, update_status_request: UpdateStatusRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/updateStatus'),
data=update_status_request.to_JSON(), bearer_token=bearer_token)
\ No newline at end of file
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.data_workflow.get_workflow_schedules_request import GetWorkflowSchedulesRequest
from osdu_api.model.http_method import HttpMethod
class DataWorkflowSchedulingClient(BaseClient):
"""
Holds the logic for interfacing with Data Workflow's scheduling api
"""
def create_workflow_schedule(self, workflow_schedule: WorkflowSchedule, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/scheduling'),
data=workflow_schedule.to_JSON(), bearer_token=bearer_token)
def list_workflow_schedules(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.data_workflow_url, '/scheduling'), bearer_token=bearer_token)
def get_workflow_schedules(self, get_workflow_schedules_request: GetWorkflowSchedulesRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/scheduling/getSchedules'),
data=get_workflow_schedules_request.to_JSON(), bearer_token=bearer_token)
def delete_workflow_schedule(self, workflow_schedule_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}{}'.format(self.data_workflow_url, '/scheduling/', workflow_schedule_name), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
from osdu_api.model.http_method import HttpMethod
class DatasetDmsClient(BaseClient):
"""
Holds the logic for interfacing with Data Registry Service's DMS api
"""
def get_storage_instructions(self, resource_type_id: str, bearer_token=None):
params = {'resourceType': resource_type_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getStorageInstructions'),
params=params, bearer_token=bearer_token)
def get_retrieval_instructions(self, record_id: str, bearer_token=None):
params = {'id': record_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getRetrievalInstructions'),
params=params, bearer_token=bearer_token)
def get_multiple_retrieval_instructions(self, get_dataset_registry_request: GetDatasetRegistryRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.dataset_url, '/getRetrievalInstructions'),
data=get_dataset_registry_request.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.create_dataset_registries_request import CreateDatasetRegistriesRequest
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
from osdu_api.model.http_method import HttpMethod
class DatasetRegistryClient(BaseClient):
"""
Holds the logic for interfacing with Data Registry Service's api
"""
def register_dataset(self, dataset_registries: CreateDatasetRegistriesRequest, bearer_token=None):
return self.make_request(method=HttpMethod.PUT, url='{}{}'.format(self.dataset_url, '/registerDataset'),
data=dataset_registries.to_JSON(), bearer_token=bearer_token)
def get_dataset_registry(self, record_id: str, bearer_token=None):
params = {'id': record_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getDatasetRegistry'),
params=params, bearer_token=bearer_token)
def get_dataset_registries(self, get_dataset_registry_request: GetDatasetRegistryRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.dataset_url, '/getDatasetRegistry'),
data=get_dataset_registry_request.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.entitlements.group import Group
from osdu_api.model.entitlements.group_member import GroupMember
class EntitlementsClient(BaseClient):
"""
Holds the logic for interfacing with Entitlement's api
"""
def get_groups_for_user(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.entitlements_url, '/groups'), bearer_token=bearer_token)
def get_group_members(self, group_email: str, limit: int, role: str, bearer_token=None):
params = {}
params['limit'] = limit
params['role'] = role
return self.make_request(method=HttpMethod.GET, url='{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members'), params=params, bearer_token=bearer_token)
def delete_group_member(self, group_email: str, member_email: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members/', member_email), bearer_token=bearer_token)
def create_group(self, group: Group, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.entitlements_url, '/groups'),
data=group.to_JSON(), bearer_token=bearer_token)
def create_group_member(self, group_email:str, group_member: GroupMember, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members'),
data=group_member.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.legal.legal_tag import LegalTag
from osdu_api.model.legal.update_legal_tag import UpdateLegalTag
from osdu_api.model.legal.legal_tag_names import LegalTagNames
from osdu_api.model.http_method import HttpMethod
class LegalClient(BaseClient):
"""
Holds the logic for interfacing with Legal's api
"""
def get_legal_tag(self, legal_tag_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}/{}'.format(self.legal_url, '/legaltags', legal_tag_name), bearer_token=bearer_token)
def create_legal_tag(self, legal_tag: LegalTag, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.legal_url, '/legaltags'),
data=legal_tag.to_JSON(), bearer_token=bearer_token)
def delete_legal_tag(self, legal_tag_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}/{}'.format(self.legal_url, '/legaltags', legal_tag_name), bearer_token=bearer_token)
def list_legal_tags(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.legal_url, '/legaltags'), bearer_token=bearer_token)
def update_legal_tag(self, update_legal_tag: UpdateLegalTag, bearer_token=None):
return self.make_request(method=HttpMethod.PUT, url='{}{}'.format(self.legal_url, '/legaltags'),
data=update_legal_tag.to_JSON(), bearer_token=bearer_token)
def get_legal_tags(self, legal_tag_names: LegalTagNames, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.legal_url, '/legaltags:batchRetrieve'),
data=legal_tag_names.to_JSON(), bearer_token=bearer_token)
def validate_legal_tags(self, legal_tag_names: LegalTagNames, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.legal_url, '/legaltags:validate'),
data=legal_tag_names.to_JSON(), bearer_token=bearer_token)
def get_legal_tag_properties(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.legal_url, '/legaltags:properties'), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#