Commit 6d1eb8ec authored by Sutton's avatar Sutton
Browse files

Merge for update

parents 00e4684d b16a9886
......@@ -187,7 +187,7 @@
identification within third-party archives.
Copyright 2020 Google LLC
Copyright 2020 Amazon
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
......
# OSDU Python SDK
To install locally:
- Make sure you have setuptools and wheel installed `python3 -m pip install --user --upgrade setuptools wheel`
- Run `python setup.py sdist bdist_wheel`
- Make sure osdu-api isn't already installed `pip uninstall osdu-api`
- Run `python -m pip install <YOUR PATH TO PACKAGE>/dist/osdu_api-0.0.1-py3-none-any.whl` make sure to substitute your machine's path in that command
Example import after installing:
`from osdu_api.storage.record_client import RecordClient`
## Contents
......@@ -87,3 +95,5 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
A package to interface with OSDU microservices
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
env:
variables:
# these are needed because unit tests check config parsing and config injects env vars
STORAGE_BASE_URL: stub
SEARCH_BASE_URL: stub
LEGAL_BASE_URL: stub
DATA_WORKFLOW_BASE_URL: stub
FILE_DMS_BASE_URL: stub
DATASET_REGISTRY_BASE_URL: stub
AUTH_TOKEN_URL: stub
CUSTOM_SCOPE: stub
ENVIRONMENT: stub
AWS_REGION: stub
phases:
install:
commands:
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
build:
commands:
# install packages
- pip install -r requirements.txt
# run basic build
- python setup.py build
# run unit tests
- cp osdu_api/test/osdu_api.ini osdu_api.ini
- python -m unittest discover osdu_api/test
- rm osdu_api.ini
# publish new artifact to code artifact
- aws codeartifact login --tool twine --domain osdu-dev --domain-owner 888733619319 --repository osdu-python
- python setup.py sdist bdist_wheel
- twine upload --skip-existing --repository codeartifact dist/osdu_api-0.0.4.tar.gz
artifacts:
files:
- "**/*"
# base-directory: ${OUTPUT_DIR}
name: test
\ No newline at end of file
# Copyright 2020 Google LLC
# Copyright 2020 Amazon
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......
# Copyright 2020 Google LLC
# Copyright 2020 Amazon
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys, os
import importlib
from configparser import SafeConfigParser
import requests
from osdu_api.model.http_method import HttpMethod
'''
Base client that is meant to be extended by service specific clients
'''
class BaseClient:
'''
Base client gets initialized with configuration values and a bearer token
based on provider-specific logic
'''
def __init__(self):
self._parse_config()
self.unauth_retries = 0
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self._refresh_service_principal_token()
'''
Example config file:
[environment]
data_partition_id=opendes
storage_url=https://[STORAGE_ENDPOINT]/api/storage/v2
search_url=https://[SEARCH_ENDPOINT]/api/search/v2
data_workflow_url=https://[WORKFLOW_ENDPOINT]/api/data-workflow/v1
file_dms_url=https://[FILE_DMS_ENDPOINT]/api/filedms/v2
dataset_registry_url=https://[DATASET_REGISTRY_URL]/api/dataset-registry/v1
[provider]
name=aws
entitlements_module_name=entitlements_client
'''
def _parse_config(self):
config_parser = SafeConfigParser(os.environ)
config_file_name = 'osdu_api.ini'
found_names = config_parser.read(config_file_name)
if config_file_name not in found_names:
raise Exception('Could not find osdu_api.ini config file')
self.data_partition_id = config_parser.get('environment', 'data_partition_id')
self.storage_url = config_parser.get('environment', 'storage_url')
self.search_url = config_parser.get('environment', 'search_url')
self.data_workflow_url = config_parser.get('environment', 'data_workflow_url')
self.file_dms_url = config_parser.get('environment', 'file_dms_url')
self.legal_url = config_parser.get('environment', 'legal_url')
self.entitlements_url = config_parser.get('environment', 'entitlements_url')
self.dataset_url = config_parser.get('environment', 'dataset_url')
self.use_service_principal = config_parser.get('environment', 'use_service_principal')
self.provider = config_parser.get('provider', 'name')
self.service_principal_module_name = config_parser.get('provider', 'service_principal_module_name')
'''
The path to the logic to get a valid bearer token is dynamically injected based on
what provider and entitlements module name is provided in the configuration yaml
'''
def _refresh_service_principal_token(self):
entitlements_client = importlib.import_module('osdu_api.provider.%s.%s' % (self.provider, self.service_principal_module_name))
self.service_principal_token = entitlements_client.get_service_principal_token()
'''
Makes a request using python's built in requests library. Takes additional headers if
necessary
'''
def make_request(self, method: HttpMethod, url: str, data = '', add_headers = {}, params = {}, bearer_token = None):
if bearer_token is not None and 'Bearer ' not in bearer_token:
bearer_token = 'Bearer ' + bearer_token
headers = {
'content-type': 'application/json',
'data-partition-id': self.data_partition_id,
'Authorization': bearer_token if bearer_token is not None else self.service_principal_token
}
if (len(add_headers) > 0):
for key, value in add_headers:
headers[key] = value
response = object
if (method == HttpMethod.GET):
response = requests.get(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.DELETE):
response = requests.delete(url=url, params=params, headers=headers, verify=False)
elif (method == HttpMethod.POST):
response = requests.post(url=url, params=params, data=data, headers=headers, verify=False)
elif (method == HttpMethod.PUT):
response = requests.put(url=url, params=params, data=data, headers=headers, verify=False)
if (response.status_code == 401 or response.status_code == 403) and self.unauth_retries < 1:
if self.use_service_principal == 'True' or self.use_service_principal == 'true':
self.unauth_retries += 1
self._refresh_service_principal_token()
self.make_request(method, url, data, add_headers, params, None)
self.unauth_retries = 0
return response
\ No newline at end of file
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
\ No newline at end of file
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.start_workflow import StartWorkflow
from osdu_api.model.data_workflow.update_status_request import UpdateStatusRequest
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with Data Workflow's api
'''
class DataWorkflowClient(BaseClient):
def start_workflow(self, start_workflow: StartWorkflow, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/startWorkflow'),
data=start_workflow.to_JSON(), bearer_token=bearer_token)
def update_status(self, update_status_request: UpdateStatusRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/updateStatus'),
data=update_status_request.to_JSON(), bearer_token=bearer_token)
\ No newline at end of file
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.data_workflow.workflow_schedule import WorkflowSchedule
from osdu_api.model.data_workflow.get_workflow_schedules_request import GetWorkflowSchedulesRequest
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with Data Workflow's scheduling api
'''
class DataWorkflowSchedulingClient(BaseClient):
def create_workflow_schedule(self, workflow_schedule: WorkflowSchedule, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/scheduling'),
data=workflow_schedule.to_JSON(), bearer_token=bearer_token)
def list_workflow_schedules(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.data_workflow_url, '/scheduling'), bearer_token=bearer_token)
def get_workflow_schedules(self, get_workflow_schedules_request: GetWorkflowSchedulesRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.data_workflow_url, '/scheduling/getSchedules'),
data=get_workflow_schedules_request.to_JSON(), bearer_token=bearer_token)
def delete_workflow_schedule(self, workflow_schedule_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}{}'.format(self.data_workflow_url, '/scheduling/', workflow_schedule_name), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with Data Registry Service's DMS api
'''
class DatasetDmsClient(BaseClient):
def get_storage_instructions(self, resource_type_id: str, bearer_token=None):
params = {'resourceType': resource_type_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getStorageInstructions'),
params=params, bearer_token=bearer_token)
def get_retrieval_instructions(self, record_id: str, bearer_token=None):
params = {'id': record_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getRetrievalInstructions'),
params=params, bearer_token=bearer_token)
def get_multiple_retrieval_instructions(self, get_dataset_registry_request: GetDatasetRegistryRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.dataset_url, '/getRetrievalInstructions'),
data=get_dataset_registry_request.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.dataset.create_dataset_registries_request import CreateDatasetRegistriesRequest
from osdu_api.model.dataset.get_dataset_registry_request import GetDatasetRegistryRequest
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with Data Registry Service's api
'''
class DatasetRegistryClient(BaseClient):
def register_dataset(self, dataset_registries: CreateDatasetRegistriesRequest, bearer_token=None):
return self.make_request(method=HttpMethod.PUT, url='{}{}'.format(self.dataset_url, '/registerDataset'),
data=dataset_registries.to_JSON(), bearer_token=bearer_token)
def get_dataset_registry(self, record_id: str, bearer_token=None):
params = {'id': record_id}
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.dataset_url, '/getDatasetRegistry'),
params=params, bearer_token=bearer_token)
def get_dataset_registries(self, get_dataset_registry_request: GetDatasetRegistryRequest, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.dataset_url, '/getDatasetRegistry'),
data=get_dataset_registry_request.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.http_method import HttpMethod
from osdu_api.model.entitlements.group import Group
from osdu_api.model.entitlements.group_member import GroupMember
'''
Holds the logic for interfacing with Entitlement's api
'''
class EntitlementsClient(BaseClient):
def get_groups_for_user(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.entitlements_url, '/groups'), bearer_token=bearer_token)
def get_group_members(self, group_email: str, limit: int, role: str, bearer_token=None):
params = {}
params['limit'] = limit
params['role'] = role
return self.make_request(method=HttpMethod.GET, url='{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members'), params=params, bearer_token=bearer_token)
def delete_group_member(self, group_email: str, member_email: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members/', member_email), bearer_token=bearer_token)
def create_group(self, group: Group, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.entitlements_url, '/groups'),
data=group.to_JSON(), bearer_token=bearer_token)
def create_group_member(self, group_email:str, group_member: GroupMember, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}{}{}'.format(self.entitlements_url, '/groups/', group_email, '/members'),
data=group_member.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.file_dms.register_files import RegisterFiles
from osdu_api.model.file_dms.get_files_delivery import GetFilesDelivery
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with File DMS's api
'''
class FileDMSClient(BaseClient):
def get_file_upload_location(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.file_dms_url, '/getFileUploadLocation'), bearer_token=bearer_token)
def register_files(self, register_files: RegisterFiles, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.file_dms_url, '/registerFiles'),
data=register_files.to_JSON(), bearer_token=bearer_token)
def get_file_delivery_instructions(self, get_files_delivery: GetFilesDelivery, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.file_dms_url, '/getFileDeliveryInstructions'),
data=get_files_delivery.to_JSON(), bearer_token=bearer_token)
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright © 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from typing import List
from osdu_api.clients.base_client import BaseClient
from osdu_api.model.legal.legal_tag import LegalTag
from osdu_api.model.legal.update_legal_tag import UpdateLegalTag
from osdu_api.model.legal.legal_tag_names import LegalTagNames
from osdu_api.model.http_method import HttpMethod
'''
Holds the logic for interfacing with Legal's api
'''
class LegalClient(BaseClient):
def get_legal_tag(self, legal_tag_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}/{}'.format(self.legal_url, '/legaltags', legal_tag_name), bearer_token=bearer_token)
def create_legal_tag(self, legal_tag: LegalTag, bearer_token=None):
return self.make_request(method=HttpMethod.POST, url='{}{}'.format(self.legal_url, '/legaltags'),
data=legal_tag.to_JSON(), bearer_token=bearer_token)
def delete_legal_tag(self, legal_tag_name: str, bearer_token=None):
return self.make_request(method=HttpMethod.DELETE, url='{}{}/{}'.format(self.legal_url, '/legaltags', legal_tag_name), bearer_token=bearer_token)
def list_legal_tags(self, bearer_token=None):
return self.make_request(method=HttpMethod.GET, url='{}{}'.format(self.legal_url, '/legaltags'), bearer_token=bearer_token)
def update_legal_tag(self, update_legal_tag: UpdateLegalTag, bearer_token=None):
return self.make_request(method=HttpMethod.PUT, url='{}{}'.format(self.legal_url, '/legaltags'),
data=update_legal_tag.to_JSON(), bearer_token=bearer_token)
def get_legal_tags(self, legal_tag_names: LegalTagNames, bearer_token=None):