diff --git a/app/README.md b/app/README.md index 6143fc94e2c249b0486bc621d7c3716a67b27a8a..e0e4a7bc679e87e3e9a33b7ccb2769634d5990ec 100644 --- a/app/README.md +++ b/app/README.md @@ -36,6 +36,7 @@ Recently policy service was migrated from Flask to FastAPI, this was done in par - A value of LOCAL will bypass using CSP and attempt to update OPA directly va `OPA_URL`. This is great for local development, testing, tiny environments (that don't have multiple OPA Pods), on-premise or for unsupported cloud environments. Currently supported values of `CLOUD_PROVIDER`: + * anthos * aws * azure * gcp @@ -48,6 +49,7 @@ Recently policy service was migrated from Flask to FastAPI, this was done in par - `CONTAINER_NAME` - used by Azure to determine which container is used for providing bundle files. Service principal running policy service needs to have write permission to contents in this bucket. - `STORAGE_ACCOUNT` - used by Azure to determine which account is used for providing bundle files. Service principal running policy service needs to have write permission to contents in this bucket. - `ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` - used by IBM (yes IBM). Please note only region `us-east-1` is currently supported. + - `MINIO_ENDPOINT`, `MINIO_SECRET_KEY`, `MINIO_ACCESS_KEY` - used by Anthos (Reference Architecture). * Useful software but not required: * Make - [GNU make utility](https://www.gnu.org/software/make/) diff --git a/app/bundles/providers/anthos/README.md b/app/bundles/providers/anthos/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f2e20b83a4dc1210d174331b0a30e38c590c4fc6 --- /dev/null +++ b/app/bundles/providers/anthos/README.md @@ -0,0 +1,7 @@ +## MinIO Variables + +### Secret Variables + +MINIO_ENDPOINT - MinIO API Endpoint URL +MINIO_ACCESS_KEY - MinIO Access Key +MINIO_SECRET_KEY - MinIO Secret Key diff --git a/app/bundles/providers/anthos/__init__.py b/app/bundles/providers/anthos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ab86bed59058046eeb793d68aa08749a0486592b --- /dev/null +++ b/app/bundles/providers/anthos/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2022 Google LLC +# Copyright 2022 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from bundles.providers.anthos.storage import MinIOBundleStorageClient as storage_client diff --git a/app/bundles/providers/anthos/storage.py b/app/bundles/providers/anthos/storage.py new file mode 100644 index 0000000000000000000000000000000000000000..8ec1d152749bb345fa74685414da6a650e6f15bb --- /dev/null +++ b/app/bundles/providers/anthos/storage.py @@ -0,0 +1,69 @@ +# Copyright 2022 Google LLC +# Copyright 2022 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import Tuple +from osdu_api.providers.blob_storage import get_client +from osdu_api.providers.types import FileLikeObject + +from bundles.storage import BundleStorageClient + +logger = logging.getLogger(__name__) + +class FileNotFound(Exception): + def __init__(self, filename: str): + self.message = f'ERROR: File {filename} was not found.' + super().__init__(self.message) + +class MinIOBundleStorageClient(BundleStorageClient): + def __init__(self) -> None: + self._client = get_client() + self._bucket_name = os.environ["POLICY_BUCKET"] + self._content_type = "application/x-gtar" + + def _get_bucket_uri(self, filename: str) -> str: + return f"s3://{self._bucket_name}/{filename}" + + def download_file(self, filename: str, file: FileLikeObject) -> Tuple[FileLikeObject, str]: + try: + uri = self._get_bucket_uri(filename) + + if self._does_file_exist(uri): + self._client.download_to_file(uri, file) + return file, uri + else: + raise FileNotFound(filename) + + except Exception as e: + logger.error(f"Failed to download file from {uri} {e}") + + def upload_file(self, name: str, file: FileLikeObject) -> str: + try: + uri = self._get_bucket_uri(name) + self._client.upload_file(uri, file, self._content_type) + return uri + except Exception as e: + logger.error(f"Failed to upload file to {uri} {e}") + + def _does_file_exist(self, uri: str) -> bool: + """Verify if a file exists in the given URI. + + :param uri: The AWS URI of the file. + :type uri: str + :return: A boolean indicating if the file exists + :rtype: bool + """ + return self._client.does_file_exist(uri) diff --git a/app/tests/anthos/anthos_jwt_client.py b/app/tests/anthos/anthos_jwt_client.py new file mode 100644 index 0000000000000000000000000000000000000000..b06d40dc2eb2a3a7de9214a9c93fd8d21ca53e0e --- /dev/null +++ b/app/tests/anthos/anthos_jwt_client.py @@ -0,0 +1,28 @@ +import os + +import jwt +import requests + + +def get_id_token(): + client_id = os.getenv('TEST_OPENID_PROVIDER_CLIENT_ID') + client_secret = os.getenv('TEST_OPENID_PROVIDER_CLIENT_SECRET') + keycloak_url = os.getenv('TEST_OPENID_PROVIDER_URL') + + data = { + 'client_id': client_id, + 'client_secret': client_secret, + 'grant_type': 'client_credentials', + 'scope': 'openid' + } + + response = requests.post(keycloak_url, data).json() + token = response['id_token'] + return token + + +def get_invalid_token(): + return jwt.encode({'some': 'payload'}, 'secret', algorithm='HS256').decode("utf-8") + +if __name__ == '__main__': + get_id_token() diff --git a/app/tests/anthos/prepare-dist.sh b/app/tests/anthos/prepare-dist.sh new file mode 100644 index 0000000000000000000000000000000000000000..868b1c466a913bd9b9af949297154d06b8872ae5 --- /dev/null +++ b/app/tests/anthos/prepare-dist.sh @@ -0,0 +1,17 @@ + +set -e + +OUTPUT_DIR="${OUTPUT_DIR:-dist}" + +INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing + +rm -rf "$INTEGRATION_TEST_OUTPUT_DIR" +mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" + +if [ ! -e requirements_dev.txt ]; then + echo "File requirements_dev.txt does not exist!" +#else +# cp requirements_dev.txt tests/ibm-test/build-aws/requirements.txt +fi +cp -r tests/anthos "${INTEGRATION_TEST_OUTPUT_DIR}" +cp -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}" diff --git a/app/tests/anthos/run-integration-tests.sh b/app/tests/anthos/run-integration-tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..8dafae6a84e031b82c9c92c5443dd49d2487449c --- /dev/null +++ b/app/tests/anthos/run-integration-tests.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# from tests/ibm/ goes up to the tests dir. +cd ../../ +# Install venv for python3 +sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv && sudo apt-get install python3-venv --fix-missing + +python3 --version +python3 -m venv env +source env/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install -r requirements.txt +python3 -m pip install wheel pytest pytest-cov + +export BUNDLE_PAUSE=30 +export DATA_PARTITION=$DATA_PARTITION_ID +export CLOUD_PROVIDER="anthos" +export OPA_URL=OSDU_GCP_OPA_URL + + +echo BUNDLE_PAUSE $BUNDLE_PAUSE +echo ENTITLEMENTS_BASE_URL $ENTITLEMENTS_BASE_URL +echo LEGAL_BASE_URL $LEGAL_BASE_URL +echo OPA_URL $OPA_URL +echo CLOUD_PROVIDER $CLOUD_PROVIDER +echo DOMAIN $DOMAIN +echo OSDU_GCP_POLICY_API $OSDU_GCP_POLICY_API +echo DATA_PARTITION $DATA_PARTITION + +svctoken=$(python3 tests/anthos/anthos_jwt_client.py) +python3 -m pytest --token="$svctoken" --service_url=$OSDU_GCP_POLICY_API --data_partition=$DATA_PARTITION diff --git a/requirements.txt b/requirements.txt index f30bcbd3be98ab69deab17c994ab61ea0c56c3ca..1239e5553bc01465790e9f2cede44148c1f5036f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ google-cloud-storage==1.40.0 # osdu dependences --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple -osdu-api[all]==0.13.0 +osdu-api[all]~=0.18.0rc1, ==0.18.* # it will install a rc-version if there is no release one.