Merge branch 'GONRG-5779_minIO_anthos' into 'master'

GONRG-5779: MinIO Anthos

See merge request !211

app/README.md

@@ -36,6 +36,7 @@ Recently policy service was migrated from Flask to FastAPI, this was done in par
     - A value of LOCAL will bypass using CSP and attempt to update OPA directly va `OPA_URL`. This is great for local development, testing, tiny environments (that don't have multiple OPA Pods), on-premise or for unsupported cloud environments.
 
     Currently supported values of `CLOUD_PROVIDER`:
+    * anthos
     * aws
     * azure
     * gcp
@@ -48,6 +49,7 @@ Recently policy service was migrated from Flask to FastAPI, this was done in par
     - `CONTAINER_NAME` - used by Azure to determine which container is used for providing bundle files. Service principal running policy service needs to have write permission to contents in this bucket.
     - `STORAGE_ACCOUNT` - used by Azure to determine which account is used for providing bundle files. Service principal running policy service needs to have write permission to contents in this bucket.
     - `ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` - used by IBM (yes IBM). Please note only region `us-east-1` is currently supported.
+    - `MINIO_ENDPOINT`, `MINIO_SECRET_KEY`, `MINIO_ACCESS_KEY` - used by Anthos (Reference Architecture).
 
 * Useful software but not required:
     * Make - [GNU make utility](https://www.gnu.org/software/make/)

app/bundles/providers/anthos/README.md

+## MinIO Variables
+
+### Secret Variables
+
+MINIO_ENDPOINT - MinIO API Endpoint URL
+MINIO_ACCESS_KEY - MinIO Access Key
+MINIO_SECRET_KEY - MinIO Secret Key

app/bundles/providers/anthos/__init__.py

+#  Copyright 2022 Google LLC
+#  Copyright 2022 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from bundles.providers.anthos.storage import MinIOBundleStorageClient as storage_client

app/bundles/providers/anthos/storage.py

+#  Copyright 2022 Google LLC
+#  Copyright 2022 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import logging
+import os
+from typing import Tuple
+from osdu_api.providers.blob_storage import get_client
+from osdu_api.providers.types import FileLikeObject
+
+from bundles.storage import BundleStorageClient
+
+logger = logging.getLogger(__name__)
+
+class FileNotFound(Exception):
+    def __init__(self, filename: str):
+        self.message = f'ERROR: File {filename} was not found.'
+        super().__init__(self.message)
+
+class MinIOBundleStorageClient(BundleStorageClient):
+    def __init__(self) -> None:
+        self._client = get_client()
+        self._bucket_name = os.environ["POLICY_BUCKET"]
+        self._content_type = "application/x-gtar"
+
+    def _get_bucket_uri(self, filename: str) -> str:
+        return f"s3://{self._bucket_name}/{filename}"
+
+    def download_file(self, filename: str, file: FileLikeObject) -> Tuple[FileLikeObject, str]:
+        try:
+            uri = self._get_bucket_uri(filename)
+
+            if self._does_file_exist(uri):
+                self._client.download_to_file(uri, file)
+                return file, uri
+            else:
+                raise FileNotFound(filename)
+            
+        except Exception as e:
+            logger.error(f"Failed to download file from {uri} {e}")
+
+    def upload_file(self, name: str, file: FileLikeObject) -> str:
+        try:
+            uri = self._get_bucket_uri(name)
+            self._client.upload_file(uri, file, self._content_type)
+            return uri
+        except Exception as e:
+            logger.error(f"Failed to upload file to {uri} {e}")
+
+    def _does_file_exist(self, uri: str) -> bool:
+            """Verify if a file exists in the given URI.
+
+            :param uri: The AWS URI of the file.
+            :type uri: str
+            :return: A boolean indicating if the file exists
+            :rtype: bool
+            """
+            return self._client.does_file_exist(uri)

app/tests/anthos/anthos_jwt_client.py

+import os
+
+import jwt
+import requests
+
+
+def get_id_token():
+    client_id = os.getenv('TEST_OPENID_PROVIDER_CLIENT_ID')
+    client_secret = os.getenv('TEST_OPENID_PROVIDER_CLIENT_SECRET')
+    keycloak_url = os.getenv('TEST_OPENID_PROVIDER_URL')
+
+    data = {
+        'client_id': client_id,
+        'client_secret': client_secret,
+        'grant_type': 'client_credentials',
+        'scope': 'openid'
+    }
+
+    response = requests.post(keycloak_url, data).json()
+    token = response['id_token']
+    return token
+
+
+def get_invalid_token():
+    return jwt.encode({'some': 'payload'}, 'secret', algorithm='HS256').decode("utf-8")
+
+if __name__ == '__main__':
+    get_id_token()

app/tests/anthos/prepare-dist.sh

+
+set -e
+
+OUTPUT_DIR="${OUTPUT_DIR:-dist}"
+
+INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing
+
+rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
+mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR"
+
+if [ ! -e requirements_dev.txt ]; then
+    echo "File requirements_dev.txt does not exist!"
+#else
+#    cp requirements_dev.txt tests/ibm-test/build-aws/requirements.txt
+fi
+cp  -r tests/anthos "${INTEGRATION_TEST_OUTPUT_DIR}"
+cp  -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}"

app/tests/anthos/run-integration-tests.sh

+#!/usr/bin/env bash
+# from tests/ibm/ goes up to the tests dir.
+cd ../../
+# Install venv for python3
+sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv && sudo apt-get install python3-venv --fix-missing
+
+python3 --version
+python3 -m venv env
+source env/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install -r requirements.txt
+python3 -m pip install wheel pytest pytest-cov
+
+export BUNDLE_PAUSE=30
+export DATA_PARTITION=$DATA_PARTITION_ID
+export CLOUD_PROVIDER="anthos"
+export OPA_URL=OSDU_GCP_OPA_URL
+
+
+echo BUNDLE_PAUSE $BUNDLE_PAUSE
+echo ENTITLEMENTS_BASE_URL $ENTITLEMENTS_BASE_URL
+echo LEGAL_BASE_URL $LEGAL_BASE_URL
+echo OPA_URL $OPA_URL
+echo CLOUD_PROVIDER $CLOUD_PROVIDER
+echo DOMAIN $DOMAIN
+echo OSDU_GCP_POLICY_API $OSDU_GCP_POLICY_API
+echo DATA_PARTITION $DATA_PARTITION
+
+svctoken=$(python3 tests/anthos/anthos_jwt_client.py)
+python3 -m pytest --token="$svctoken" --service_url=$OSDU_GCP_POLICY_API --data_partition=$DATA_PARTITION

requirements.txt

@@ -18,4 +18,4 @@ google-cloud-storage==1.40.0
 
 # osdu dependences
 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple 
-osdu-api[all]==0.13.0
+osdu-api[all]~=0.18.0rc1, ==0.18.* # it will install a rc-version if there is no release one.