diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000000000000000000000000000000..5b462cccdadf92874f22e8ef07043baace8eebf9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,4 @@ +root = true + +[*] +insert_final_newline = true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6556a833bb6bb6f7521abf6d89ec79b14e05646d..178988543670f91b876164cd6dd31d277d7b7cb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: PIP_REQUIREMENTS: "requirements.txt requirements_dev.txt app/requirements.txt" PYTHONPATH: app - PYTEST_DIR: app/tests + PYTEST_DIR: app/tests/unit AWS_BUILD_SUBDIR: build/aws/build-aws # AWS_TEST_SUBDIR: testing/storage-test-aws diff --git a/NOTICE b/NOTICE index 6f5876c4f7660ff9a7f853472324c7d6782642c7..b980f71ed4e9414dd1b998356d02aff0421fee68 100644 --- a/NOTICE +++ b/NOTICE @@ -42,7 +42,8 @@ The following software have components provided under the terms of this license: - mock (from http://mock.readthedocs.org/en/latest/, https://github.com/testing-cabal/mock) - packaging (from https://github.com/pypa/packaging) -- pyasn1 (from http://sourceforge.net/projects/pyasn1/) +- protobuf (from https://developers.google.com/protocol-buffers/) +- pyasn1 (from http://sourceforge.net/projects/pyasn1/, https://github.com/etingof/pyasn1) - pyasn1-modules (from http://sourceforge.net/projects/pyasn1/, https://github.com/etingof/pyasn1-modules) ======================================================================== @@ -60,7 +61,7 @@ The following software have components provided under the terms of this license: - httpx (from https://github.com/encode/httpx) - idna (from https://github.com/kjd/idna) - isodate (from http://cheeseshop.python.org/pypi/isodate) -- itsdangerous (from http://github.com/mitsuhiko/itsdangerous) +- itsdangerous (from http://github.com/mitsuhiko/itsdangerous, https://palletsprojects.com/p/itsdangerous/) - jinja2 - mock (from https://github.com/testing-cabal/mock) - oauthlib (from https://github.com/idan/oauthlib, https://github.com/oauthlib/oauthlib) @@ -100,7 +101,7 @@ The following software have components provided under the terms of this license: - MarkupSafe (from https://palletsprojects.com/p/markupsafe/) - Werkzeug (from https://palletsprojects.com/p/werkzeug/) - click (from http://github.com/mitsuhiko/click) -- itsdangerous (from http://github.com/mitsuhiko/itsdangerous) +- itsdangerous (from http://github.com/mitsuhiko/itsdangerous, https://palletsprojects.com/p/itsdangerous/) - jinja2 - requests-oauthlib (from https://github.com/requests/requests-oauthlib) @@ -138,7 +139,7 @@ The following software have components provided under the terms of this license: - azure-common (from https://github.com/Azure/azure-sdk-for-python) - azure-core (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core) - azure-identity (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity, https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/identity/azure-identity) -- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets) +- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/keyvault/azure-keyvault-secrets, https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets) - azure-storage-blob (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob) - botocore (from https://github.com/boto/botocore) - cachetools (from https://github.com/tkem/cachetools/) diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..66a8acd6767ff889d868ec263ac9eaad8ec7b9b9 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Policy service + +Policy service is used for management and evaluation of dynamic policies in OSDU. + +## Dependencies +Policy service has a runtime dependency on Open Policy Agent [OPA](https://openpolicyagent.org/). +It also has a runtime dependency on OSDU entitlements service for API authorization. + +## Environmental variables +_ENTITLEMENTS_BASE_URL_ and _LEGAL_BASE_URL_ environmental variable must be set to run the service. The values provided should be of format _scheme://host[:port]_ and not include any part of path (e.g., https://entitlements.osdu.compay.com). + +## API authorization +To read or evaluate policies, user calling the policy service, must be a member of _service.policy.user_ or _service.policy.admin_ group. + +To create, update, or delete the policies, user must be a member of _service.policy.admin_ group. + +User can be added to these groups by using entitlements service. + +For information on how to run the policy service locally: +[Policy Service README](./app/README.md) + +There is also a Policy Service Admin CLI: +[Policy Service Admin CLI](./frontend/admincli/README.md) diff --git a/app/Makefile b/app/Makefile index 305de088bc54976e3467ea0abfa2b286aeb7efe6..964cb24d25f3a79e0ac990ecb60e4adf7377ebd9 100644 --- a/app/Makefile +++ b/app/Makefile @@ -26,8 +26,7 @@ build_docker: cd .. && docker build --network host -t $(IMAGE_NAME):$(TAG) -f app/Dockerfile . build_adminui: - cd ../frontend/adminui && ng build - + cd ../frontend/adminui && npm install && ng build # run policy-service locally in docker with gcp backend, assumes opa running locally run: gcp_set_token diff --git a/app/README.md b/app/README.md index b69cbea29919e503bea8980011d8d53f2e5b325a..3e5b718bb82e38215a2ef5dccbf87ec44071744d 100644 --- a/app/README.md +++ b/app/README.md @@ -53,6 +53,8 @@ Policy service and its integration with the rest of OSDU is an experimental feat * ttab - [programmatically open a new terminal tab](https://www.npmjs.com/package/ttab) * envsubst - [GNU gettext utilities](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) +* Port - the default for the policy service is 8080. In Kubernetes you can still expose this to port 80. + ## Testing Policy Service The [test directory](./test) contains pytest unit and integration tests and their associated data files. * Unit tests assume no services are available. @@ -126,6 +128,4 @@ For more information on [translate API](./translate/README.md). ``` ## Todo / Planned work: -* Postman collections updated * Admin UI -* A reference kubernetes helm chart/yaml will be available in future releases. diff --git a/app/api/policy_read_api.py b/app/api/policy_read_api.py index c4d8dc410b7be8127f67992d31df89fc96973c79..fdea9a2f8ed4d8263f0bbf5643f3c44286ef961f 100644 --- a/app/api/policy_read_api.py +++ b/app/api/policy_read_api.py @@ -51,7 +51,13 @@ async def fetch_all_policies_that_match_partition(data_partition_id: Union[str, logging.info(f"data_partition_id: {data_partition_id}") opa_response = opa.fetch_opa_policies_direct() if not opa_response.ok: - raise HTTPException(status_code=opa_response.status_code, detail="Unexpected result from OPA") + cloud_provider = os.environ.get('CLOUD_PROVIDER') + if cloud_provider is None or cloud_provider == conf.MOCK: + opa_response = OpaResponse() + logger.critical(f"Error: CLOUD_PROVIDER ENV VAR not set / Mocking results for policies") + opa_response.json = {"result": {}} + else: + raise HTTPException(status_code=opa_response.status_code, detail="Unexpected result from OPA") if conf.USE_BUNDLES and 'result' in opa_response.json: logger.debug(f"OPA return {opa_response.json}") policies = opa_response.json @@ -169,4 +175,4 @@ async def fetch_partition_policy_directly_from_opa( status_code=opa_response.status_code, detail=f"Unexpected result from OPA {opa_response.message}", #headers=headers - ) \ No newline at end of file + ) diff --git a/app/bundles/providers/azure/helper.py b/app/bundles/providers/azure/helper.py deleted file mode 100644 index 1fd223175eb0932d38164d7113c517257d03a399..0000000000000000000000000000000000000000 --- a/app/bundles/providers/azure/helper.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -from azure.identity import DefaultAzureCredential -from azure.keyvault.secrets import SecretClient - - -class Helper(object): - - KEY_VAULT_URI = os.getenv("KEYVAULT_URI") - - def get_key_vault_secret(self, key: str): - credential = DefaultAzureCredential() - secret_client = SecretClient( - vault_url=Helper.KEY_VAULT_URI, - credential=credential - ) - return secret_client.get_secret(key).value - - def get_storage_connection_string(self): - return self.get_key_vault_secret("airflow-storage-connection") diff --git a/app/bundles/providers/azure/storage.py b/app/bundles/providers/azure/storage.py index d86a4d5e359a3654076e062a1c8e9146055dfcea..f2c2cd0c1216ed42ee41d0ca8f5f9ed2aacda429 100644 --- a/app/bundles/providers/azure/storage.py +++ b/app/bundles/providers/azure/storage.py @@ -5,7 +5,8 @@ from typing import Tuple from osdu_api.providers.types import FileLikeObject from bundles.storage import BundleStorageClient from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient -from helper import Helper +from azure.identity import DefaultAzureCredential +from azure.keyvault.secrets import SecretClient logger = logging.getLogger(__name__) @@ -36,7 +37,23 @@ class AzureBundleStorageClient(BundleStorageClient): uri = self._get_uri(name) blob_client = self.client.get_blob_client(container=self.container_name, blob=name) #not sure how to set content type - blob_client.upload_blob(file.read(), blob_type="BlockBlob") + blob_client.upload_blob(file.read(), overwrite=True, blob_type="BlockBlob") return uri except Exception as e: logger.error(f"Failed to upload file to {uri}") + + +class Helper(object): + + KEY_VAULT_URI = os.getenv("KEYVAULT_URI") + + def get_key_vault_secret(self, key: str): + credential = DefaultAzureCredential() + secret_client = SecretClient( + vault_url=Helper.KEY_VAULT_URI, + credential=credential + ) + return secret_client.get_secret(key).value + + def get_storage_connection_string(self): + return self.get_key_vault_secret("airflow-storage-connection") diff --git a/app/tests/integration/test_integration_020_put.py b/app/tests/integration/test_integration_020_put.py index 6d644e01da90e6fe19ef515de5cf600d1ab52d99..3a8bb52885a880d7d07928008babd050fbd64557 100644 --- a/app/tests/integration/test_integration_020_put.py +++ b/app/tests/integration/test_integration_020_put.py @@ -27,7 +27,7 @@ import testlib # override dependency injection for authentication to entitlement service from override_depends import override_require_authorized_user, set_authorize_session, ADMIN_ONLY_SVC, USER_AND_ADMIN_SVC, USER_ONLY_SVC, OTHER_ONLY_SVC -TEST_DATA_DIR = Path(__file__).resolve().parent / 'templates' +TEST_DATA_DIR = Path(__file__).resolve().parent / '../templates' client = TestClient(app) @@ -43,4 +43,4 @@ def test_put_policies_test_data_mock(token, data_partition): @pytest.mark.dependency(depends=["require_token"]) def test_put_policies_test_data_service_url(token, data_partition, bundle_pause, service_url): assert token is not None, "No token provided on command line" - testlib.put_policies_test_data(client=requests, token=token, data_partition=data_partition, service_url=service_url) \ No newline at end of file + testlib.put_policies_test_data(client=requests, token=token, data_partition=data_partition, service_url=service_url) diff --git a/app/tests/integration/test_integration_030_get.py b/app/tests/integration/test_integration_030_get.py index d9e6a8e9e12d235588ec4a01782b2b9ec0166200..26928678017ade31e0936fe0cf28abc69df08bc0 100644 --- a/app/tests/integration/test_integration_030_get.py +++ b/app/tests/integration/test_integration_030_get.py @@ -123,7 +123,7 @@ def test_get_diag_policies_mock(token, data_partition): #print(f"text: {r.text}",file=sys.stderr) assert r.status_code == 200, "About Page is available" p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -153,7 +153,7 @@ def test_get_diag_policies_service_url(token, data_partition, service_url): #print(f"text: {r.text}",file=sys.stderr) assert r.status_code == 200, "diag policies API is not available" p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -162,7 +162,7 @@ def test_get_diag_policies_service_url(token, data_partition, service_url): assert num_tests >= 7, "At least 7 polices were tested" @pytest.mark.dependency(depends=["require_token"]) -def test_fetch_policies(token, data_partition): +def test_fetch_policies_advanced(token, data_partition): """ get partition policies Test get /policies/osdu/partition// using built-in @@ -170,7 +170,7 @@ def test_fetch_policies(token, data_partition): """ num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -199,7 +199,7 @@ def get_osdu_partition_policies(token, data_partition, service_url=False): #pytest.skip("Skipping diagnostic test") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -224,4 +224,4 @@ def get_osdu_partition_policies(token, data_partition, service_url=False): assert f"package osdu.partition[\\\"{data_partition}\\\"].{policy_id_short}" in r.text, f"expect result for {filename}" print(r.text) num_tests = num_tests + 1 - assert num_tests >= 7, "At least 7 polices were tested" \ No newline at end of file + assert num_tests >= 7, "At least 7 polices were tested" diff --git a/app/tests/integration/test_integration_034_compile.py b/app/tests/integration/test_integration_034_compile.py index c0070ee8400dc6fd5322f21a619712b50e1f5def..5c970f10c3765b73242d41d481e00ac87dd950b8 100644 --- a/app/tests/integration/test_integration_034_compile.py +++ b/app/tests/integration/test_integration_034_compile.py @@ -62,7 +62,7 @@ def test_compile(token, data_partition): num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.json' file_path = os.path.join(template_datadir, filename) @@ -79,4 +79,4 @@ def test_compile(token, data_partition): bdata = data.encode('utf-8') files = {'file': (input_json, bdata)} - post_request(token=token, data_partition=data_partition, files=files) \ No newline at end of file + post_request(token=token, data_partition=data_partition, files=files) diff --git a/app/tests/integration/test_integration_035_eval.py b/app/tests/integration/test_integration_035_eval.py index 84c01d3ff6c00e8b984b671cba0faaad9f76257a..799f57dfaafdd3448eb3770c02f5df42c5003234 100644 --- a/app/tests/integration/test_integration_035_eval.py +++ b/app/tests/integration/test_integration_035_eval.py @@ -88,7 +88,7 @@ def test_eval_dataauthz(token, data_partition, service_url): print(f"Using Legal tag: {legal_tag}") p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.json' file_path = os.path.join(template_datadir, filename) @@ -134,4 +134,4 @@ def get_legal_tags(token, data_partition): assert x == tags, f"Expected to get {tags} from legal {retval}" legal_tags.append(tags) - return legal_tags \ No newline at end of file + return legal_tags diff --git a/app/tests/integration/test_integration_040_delete.py b/app/tests/integration/test_integration_040_delete.py index ef9465efc514ad48781c3fa653e00a36e2da14b2..9ee08df5b50f82711a90a78f2c477f70cd612312 100644 --- a/app/tests/integration/test_integration_040_delete.py +++ b/app/tests/integration/test_integration_040_delete.py @@ -43,7 +43,7 @@ def test_delete_osdu_partition_policies_service_url(token, data_partition, bundl #pytest.skip("Skipping diagnostic test") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -56,4 +56,4 @@ def test_delete_osdu_partition_policies_service_url(token, data_partition, bundl assert "result" in r.text, f"result not expected {r.text}" num_tests = num_tests + 1 assert num_tests >= 7, "At least 7 polices were tested" - #time.sleep(int(bundle_pause)) \ No newline at end of file + #time.sleep(int(bundle_pause)) diff --git a/app/tests/integration/test_integration_045_delete.py b/app/tests/integration/test_integration_045_delete.py index e02ca194e67375403834577b2c0b4d095f268010..e6166ac8548ae5262f46ae792de0295427a17505 100644 --- a/app/tests/integration/test_integration_045_delete.py +++ b/app/tests/integration/test_integration_045_delete.py @@ -43,7 +43,7 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur #pytest.skip("Skipping delete") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -55,4 +55,4 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur assert r.status_code == 404, f"Expect 404_NOT_FOUND. Delete of {filename} {url} should already be deleted" assert "not_found" in r.text, f"expect not found message {r.text}" num_tests = num_tests + 1 - assert num_tests >= 7, "At least 7 polices were tested" \ No newline at end of file + assert num_tests >= 7, "At least 7 polices were tested" diff --git a/app/tests/testlib.py b/app/tests/testlib.py index a41c796a281c1af5a14720a586225e03d59ccf78..66006cfce8682a36f20d6343e80a1e81fd55664c 100644 --- a/app/tests/testlib.py +++ b/app/tests/testlib.py @@ -67,4 +67,4 @@ def get_list_legal_tags(token, data_partition): 'data-partition-id': data_partition}) #print(json.dumps(r.json(), indent=4)) - return r.json()["legalTags"] \ No newline at end of file + return r.json()["legalTags"] diff --git a/build/aws/build-aws/buildspec.yaml b/build/aws/build-aws/buildspec.yaml index 0f90f26a014d389d0a3bb37c72f1b15a1250c1ea..32d2f03c5a7b42476f193a9a97f60df47756d9b3 100644 --- a/build/aws/build-aws/buildspec.yaml +++ b/build/aws/build-aws/buildspec.yaml @@ -47,6 +47,10 @@ phases: - ./tests/aws/build-aws/prepare-dist.sh - echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded - printenv + + - echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--" + - rsync -r devops/aws/* "${OUTPUT_DIR:-dist}" + - echo "Logging into Docker Hub..." - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} - echo "Building docker image..." diff --git a/deployment/scripts/aws/AwsPolicyBootstrap.py b/deployment/scripts/aws/AwsPolicyBootstrap.py index bf3dd4d9dc11c31419774b6d3526b80296ca389d..ec5b9a55c6c7ea26aec09e47aca0b2515d568285 100644 --- a/deployment/scripts/aws/AwsPolicyBootstrap.py +++ b/deployment/scripts/aws/AwsPolicyBootstrap.py @@ -205,5 +205,5 @@ if __name__ == '__main__': policy_util.upsert_instance_policy_bundle(DEFAULT_POLICIES_PATH) policy_util.upsert_partition_policy_bundles( DEFAULT_POLICY_TEMPLATES_PATH) - policy_util.upsert_opa_config_map( - DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_PATH, DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_FILENAME) + # policy_util.upsert_opa_config_map( + # DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_PATH, DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_FILENAME) diff --git a/deployment/scripts/aws/bootstrap.sh b/deployment/scripts/aws/bootstrap.sh index 21ba3972c4e0ec4689f149ef757aae1bcd8c7c39..16dc73f1675a8540a3732478fd8a2e1b19532850 100644 --- a/deployment/scripts/aws/bootstrap.sh +++ b/deployment/scripts/aws/bootstrap.sh @@ -33,6 +33,9 @@ if [ -z "$PARTITIONS" ]; then export PARTITIONS=osdu fi +# Create and upload Policy bundles to S3 pip3 install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt - python $DEPLOYMENTS_BASE_DIR/scripts/aws/AwsPolicyBootstrap.py $AWS_REGION $AWS_SERVICE_NAMESPACE $POLICY_CONFIG_MAP_NAME $POLICY_BUCKET --partitions $PARTITIONS +# Update helm ConfigMap +# helm dependency update $DEPLOYMENTS_BASE_DIR/scripts/aws/opa-config +# helm upgrade opa-config $DEPLOYMENTS_BASE_DIR/scripts/aws/opa-config -i -n $AWS_SERVICE_NAMESPACE --set global.region=$AWS_REGION --set nameOverride=$POLICY_CONFIG_MAP_NAME --set global.policyBucket=$POLICY_BUCKET --set partitions=${PARTITIONS} --debug --dry-run \ No newline at end of file diff --git a/devops/aws/chart/.helmignore b/devops/aws/chart/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..6e238cb1cbd9e264bf7790a760830304695e22a1 --- /dev/null +++ b/devops/aws/chart/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Build files +*.html +build-info.json \ No newline at end of file diff --git a/devops/aws/chart/Chart.yaml b/devops/aws/chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a0956c9aa96300134d54693979109527775bbb1 --- /dev/null +++ b/devops/aws/chart/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: "os-policy" +version: 0.1.0 +kubeVersion: "v1.21.x-x-x" +description: Policy Service Helm Chart for Kubernetes +type: application +appVersion: __VERSION__ +dependencies: + - name: osdu-aws-lib + version: 0.1.0 + repository: __HELM_REPO__/osdu-aws-lib/ +deprecated: false diff --git a/devops/aws/chart/README.md b/devops/aws/chart/README.md new file mode 100644 index 0000000000000000000000000000000000000000..708ad099635eba52e356c083ae4b018422d6ab01 --- /dev/null +++ b/devops/aws/chart/README.md @@ -0,0 +1,57 @@ +# Helm Chart + +## Introduction +The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager. + +## Prerequisites +The below software must be installed before continuing: +* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/) +* [Helm ^3.7.1](https://helm.sh/docs/intro/install/) +* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3) + +Additionally, an OSDU on AWS environment must be deployed. + +## Installation/Updating +To install or update the service application by executing the following command in the CHART folder: + +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] +``` + +To observe the Kubernetes resources before deploying them using the command: +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug +``` + +To observe the history of the current release, use the following command: +```bash +helm history [RELEASE_NAME] -n [NAMESPACE] +``` + +To revert to a previous release, use the following command: +```bash +helm rollback [RELEASE] [REVISION] -n [NAMESPACE] +``` + +### Customizing the Deployment +It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration: + +| Name | Example Value | Description | Type | Required | +| --- | ------------- | ----------- | ---- | -------- | +| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes | +| `global.resourcePrefix` | `osdu` | The resource prefix of the OSDU deployment. | str | yes | +| `global.policyBucket` | `osdu-policy-bucket` | The name of the S3 bucket containing policies. Can be found in the SSM parameter: /osdu/${resourcePrefix}/policy/s3DataBucket/name. | str | yes | +| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no | +| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no | +| `replicaCount` | `1` | The number of pod replicas to be deployed | int | no | +| `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no | +| `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no | +| `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no | + +## Uninstalling the Chart +To uninstall the helm release: + +```bash +helm uninstall [RELEASE] -n [NAMESPACE] --keep-history +``` \ No newline at end of file diff --git a/devops/aws/chart/templates/NOTES.txt b/devops/aws/chart/templates/NOTES.txt new file mode 100644 index 0000000000000000000000000000000000000000..f15afa919efb6cf0dc53f0d70283f9ae0dbbcbbb --- /dev/null +++ b/devops/aws/chart/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace +{{- if (.Values.service).apiPath }} +Application URL available at path {{ .Values.service.apiPath }} +{{- end }} \ No newline at end of file diff --git a/devops/aws/chart/templates/authorizationpolicy.yaml b/devops/aws/chart/templates/authorizationpolicy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..719458418874e7ffba8a776d9cae2ff885509177 --- /dev/null +++ b/devops/aws/chart/templates/authorizationpolicy.yaml @@ -0,0 +1,5 @@ +{{- template "common.authzpolicy" (list . "os-policy.authzpolicy") -}} +{{- define "os-policy.authzpolicy" -}} +## Define overrides for the service authorization policy resource here +{{- end -}} + diff --git a/devops/aws/chart/templates/deployment.yaml b/devops/aws/chart/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..854d1b541b88bea72ff9c1e0d76d37b7a708eb14 --- /dev/null +++ b/devops/aws/chart/templates/deployment.yaml @@ -0,0 +1,4 @@ +{{- template "common.deployment" (list . "os-policy.deployment") -}} +{{- define "os-policy.deployment" -}} +## Define overrides for the service deployment resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/destinationrule.yaml b/devops/aws/chart/templates/destinationrule.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc5e6d3a74af502476c3e3939c85237f745e7e46 --- /dev/null +++ b/devops/aws/chart/templates/destinationrule.yaml @@ -0,0 +1,4 @@ +{{- template "common.destinationrule" (list . "os-policy.destinationrule") -}} +{{- define "os-policy.destinationrule" -}} +## Define overrides for the service destination rule resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/hpa.yaml b/devops/aws/chart/templates/hpa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ad82bf40d77460c46507913b38defe03baee8e1 --- /dev/null +++ b/devops/aws/chart/templates/hpa.yaml @@ -0,0 +1,4 @@ +{{- template "common.hpa" (list . "os-policy.hpa") -}} +{{- define "os-policy.hpa" -}} +## Define overrides for the service HPA resource here +{{- end -}} diff --git a/devops/aws/chart/templates/service.yaml b/devops/aws/chart/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2030a3f7422eed95b678a9ad750605442e03a271 --- /dev/null +++ b/devops/aws/chart/templates/service.yaml @@ -0,0 +1,4 @@ +{{- template "common.service" (list . "os-policy.service") -}} +{{- define "os-policy.service" -}} +## Define overrides for the service resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/serviceaccount.yaml b/devops/aws/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e12ecef6fd2b98374a5b419ced4806979ea6c65 --- /dev/null +++ b/devops/aws/chart/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +{{- template "common.serviceaccount" (list . "os-policy.serviceaccount") -}} +{{- define "os-policy.serviceaccount" -}} +## Define overrides for the service's service account resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/tests/test-connection.yaml b/devops/aws/chart/templates/tests/test-connection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f341212ea253eced15fd153dd894287341d8db93 --- /dev/null +++ b/devops/aws/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.fullname" . }}-test-connection" + labels: + {{- include "common.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/devops/aws/chart/templates/virtualservice.yaml b/devops/aws/chart/templates/virtualservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5453ae0de3715ac59370e34aa556f453b353c734 --- /dev/null +++ b/devops/aws/chart/templates/virtualservice.yaml @@ -0,0 +1,4 @@ +{{- template "common.virtualservice" (list . "os-policy.virtualservice") -}} +{{- define "os-policy.virtualservice" -}} +## Define overrides for the service's virtual service resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/values.schema.json b/devops/aws/chart/values.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..c3bc0eb63c7dbdc49322be085c6a65f1be4938b7 --- /dev/null +++ b/devops/aws/chart/values.schema.json @@ -0,0 +1,271 @@ +{ + "$schema": "osdu-aws-services-helm", + "type": "object", + "default": {}, + "required": [ + "image", + "imagePullPolicy", + "service", + "podAnnotations", + "replicaCount", + "serviceAccountRole", + "securityContext" + ], + "properties": { + "image": { + "type": "string", + "title": "The service container image URL" + }, + "imagePullPolicy": { + "type": "string", + "pattern": "^(Always|Never|IfNotPresent)$", + "default": "Always" + }, + "service": { + "type": "object", + "required": [ + "type", + "port", + "apiPath" + ], + "properties": { + "type": { + "type": "string", + "default": "ClusterIP" + }, + "port": { + "type": "integer", + "default": 8080, + "title": "The port the service is hosted on", + "examples": [ + 8080, + 443, + 80 + ] + }, + "apiPath": { + "type": "string", + "title": "The main API path", + "examples": [ + "/api/service/v1/" + ] + }, + "readinessProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + }, + "livenessProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + }, + "startupProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + } + }, + "examples": [{ + "type": "ClusterIP", + "port": 8080, + "apiPath": "/api/service/v1/", + "readinessProbeHttpPath": "/api/service/v1/actuator/health", + "livenessProbeHttpPath": "/api/service/v1/actuator/health", + "startupProbeHttpPath": "/api/service/v1/actuator/health" + }] + }, + "serviceParameters": { + "type": "array", + "title": "The secret parameters consumed by the service", + "items": { + "type": "object", + "required": [ + "objectName", + "objectType", + "objectAlias" + ], + "properties": { + "objectName": { + "type": "string", + "title": "The SSM or secrets parameter name", + "examples": [ + "/osdu/resource-prefix/redis/redisauthtoken", + "/osdu/resource-prefix/redis-core/end-point" + ] + }, + "objectType": { + "type": "string", + "title": "The type of parameter", + "pattern": "^(secretsmanager|ssmparameter|)$" + }, + "objectAlias": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "CACHE_CLUSTER_KEY", + "CACHE_CLUSTER_ENDPOINT" + ] + } + }, + "examples": [{ + "objectName": "/osdu/resource-prefix/redis/redisauthtoken", + "objectType": "secretsmanager", + "objectAlias": "CACHE_CLUSTER_KEY" + }, + { + "objectName": "/osdu/resource-prefix/redis-core/end-point", + "objectType": "ssmparameter", + "objectAlias": "CACHE_CLUSTER_ENDPOINT" + }] + } + }, + "environmentVariables": { + "type": "array", + "title": "The environment variables consumed by the service", + "items": { + "type": "object", + "title": "An environment variable definition", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "APPLICATION_PORT", + "AWS_REGION" + ] + }, + "value": { + "type": "string", + "title": "The value of the environment variable" + } + }, + "examples": [{ + "name": "APPLICATION_PORT", + "value": "8080" + }, + { + "name": "AWS_REGION", + "value": "us-east-1" + }] + } + }, + "podAnnotations": { + "type": "object", + "default": {} + }, + "replicaCount": { + "type": "integer", + "default": 1, + "title": "The number of pod replicas" + }, + "maxConnections": { + "type": "integer", + "default": 200, + "title": "The max number of service connections" + }, + "resources": { + "type": "object" + }, + "autoscaling": { + "type": "object" + }, + "serviceAccountRole": { + "type": "string", + "default": "", + "title": "The service account IAM role" + }, + "cors": { + "type": "object", + "default": {}, + "title": "The cors Schema", + "required": [ + "maxAge", + "allowCredentials", + "allowMethods", + "allowHeaders" + ], + "properties": { + "maxAge": { + "type": "string", + "default": "60m", + "title": "The max age of the request" + }, + "allowCredentials": { + "type": "boolean", + "default": true, + "title": "The allowCredentials Schema" + }, + "allowMethods": { + "type": "array", + "title": "A list of supported API request methods", + "items": { + "type": "string", + "title": "Supported API request methods", + "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$" + }, + "examples": [ + ["POST", + "GET", + "PATCH", + "DELETE" + ] + ] + }, + "allowHeaders": { + "type": "array", + "title": "A list of supported headers", + "items": { + "type": "string" + }, + "examples": [ + ["Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + ] + } + }, + "examples": [{ + "maxAge": "60m", + "allowCredentials": true, + "allowMethods": [ + "POST", + "GET", + "PATCH", + "DELETE" + ], + "allowHeaders": [ + "Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + }] + }, + "securityContext": { + "type": "object", + "default": {} + }, + "allowedPrincipals": { + "type": "array", + "title": "A list of allowed service principals", + "items": { + "type": "string", + "title": "Allowed principal", + "examples": [ + "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account", + "cluster.local/ns/osdu-services/sa/compliance-queue" + ] + } + } + } +} \ No newline at end of file diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bff7371012cf5ff4c77de265d470a54e0fdd091 --- /dev/null +++ b/devops/aws/chart/values.yaml @@ -0,0 +1,75 @@ + +# Service Config +image: __CONTAINER__ +imagePullPolicy: IfNotPresent +service: + type: ClusterIP + port: 8080 + apiPath: /api/policy/v1/ + readinessProbeHttpPath: /api/policy/v1/health + livenessProbeHttpPath: /api/policy/v1/health + +environmentVariables: + - name: APPLICATION_PORT + value: "{{ .Values.service.port }}" + - name: AWS_REGION + value: "{{ .Values.global.region }}" + - name: ENVIRONMENT + value: "{{ .Values.global.resourcePrefix }}" + - name: JAVA_OPTS + value: "-Xms538M -Xmx900M" + - name: LOG_LEVEL + value: "{{ default `INFO` .Values.global.logLevel }}" + - name: SSM_ENABLED + value: "True" + - name: SSL_ENABLED + value: "false" + - name: CLOUD_PROVIDER + value: "aws" + - name: POLICY_BUCKET + value: "{{ .Values.global.policyBucket }}" + - name: USE_BUNDLES + value: "true" + - name: ENTITLEMENTS_BASE_URL + value: "http://os-entitlements:8080" + - name: ENTITLEMENTS_BASE_PATH + value: /api/entitlements/v2/groups + - name: LEGAL_BASE_URL + value: http://os-legal:8080 + - name: OPA_URL + value: http://opa-agent +podAnnotations: {} + +# Resource Config +replicaCount: 1 +resources: + limits: + memory: 900M + requests: + cpu: 500m + memory: 300M +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Security Config +serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/{{ .Values.global.resourcePrefix }}-{{ include "common.name" . }} +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 +allowedPrincipals: + - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account + - cluster.local/ns/aws-binary-dms/sa/binary-dms + - cluster.local/ns/osdu-airflow/sa/airflow-dag-upload + - cluster.local/ns/osdu-ingest/sa/os-data-workflow + - cluster.local/ns/osdu-ingest/sa/os-ingestion-workflow + - cluster.local/ns/osdu-seismic-ddms/sa/os-seismic-store + - cluster.local/ns/osdu-well-delivery/sa/os-welldelivery + - cluster.local/ns/osdu-wellbore-ddms/sa/os-wellbore-ddms diff --git a/devops/aws/opa/.helmignore b/devops/aws/opa/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..6e238cb1cbd9e264bf7790a760830304695e22a1 --- /dev/null +++ b/devops/aws/opa/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Build files +*.html +build-info.json \ No newline at end of file diff --git a/devops/aws/opa/Chart.yaml b/devops/aws/opa/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..174b5014f2923de7ef96c7c9fffb62f50c375620 --- /dev/null +++ b/devops/aws/opa/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: "opa-agent" +version: 0.1.0 +kubeVersion: "v1.21.x-x-x" +description: OPA Agent Helm Chart for Kubernetes +type: application +appVersion: __VERSION__ +dependencies: + - name: osdu-aws-lib + version: 0.1.0 + repository: __HELM_REPO__/osdu-aws-lib/ +deprecated: false diff --git a/devops/aws/opa/README.md b/devops/aws/opa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0679de921d439e3f02f809219b30f3b0c4f698ee --- /dev/null +++ b/devops/aws/opa/README.md @@ -0,0 +1,53 @@ +# Helm Chart + +## Introduction +The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager. + +## Prerequisites +The below software must be installed before continuing: +* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/) +* [Helm ^3.7.1](https://helm.sh/docs/intro/install/) +* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3) + +Additionally, an OSDU on AWS environment must be deployed. + +## Installation/Updating +To install or update the service application by executing the following command in the CHART folder: + +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] +``` + +To observe the Kubernetes resources before deploying them using the command: +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug +``` + +To observe the history of the current release, use the following command: +```bash +helm history [RELEASE_NAME] -n [NAMESPACE] +``` + +To revert to a previous release, use the following command: +```bash +helm rollback [RELEASE] [REVISION] -n [NAMESPACE] +``` + +### Customizing the Deployment +It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration: + +| Name | Example Value | Description | Type | Required | +| --- | ------------- | ----------- | ---- | -------- | +| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes | +| `global.policyBucket` | `osdu-policy-bucket` | The name of the S3 bucket containing policies. Can be found in the SSM parameter: /osdu/${resourcePrefix}/policy/s3DataBucket/name. | str | yes | +| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no | +| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no | +| `partitions` | `{osdu,opndes}` | A list of partitions with policies | array[str] | no | + +## Uninstalling the Chart +To uninstall the helm release: + +```bash +helm uninstall [RELEASE] -n [NAMESPACE] --keep-history +``` \ No newline at end of file diff --git a/devops/aws/opa/templates/NOTES.txt b/devops/aws/opa/templates/NOTES.txt new file mode 100644 index 0000000000000000000000000000000000000000..f15afa919efb6cf0dc53f0d70283f9ae0dbbcbbb --- /dev/null +++ b/devops/aws/opa/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace +{{- if (.Values.service).apiPath }} +Application URL available at path {{ .Values.service.apiPath }} +{{- end }} \ No newline at end of file diff --git a/devops/aws/opa/templates/configmap.yaml b/devops/aws/opa/templates/configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d3598ef51e984db74980b35c72f6bbef43aa113 --- /dev/null +++ b/devops/aws/opa/templates/configmap.yaml @@ -0,0 +1,23 @@ +{{- template "common.configmap" (list . "opa-agent.configmap") -}} +{{- define "opa-agent.configmap" -}} +metadata: + name: opa-bundle-server-config +data: + config: | + services: + s3: + url: https://{{ .Values.global.policyBucket }}.s3.amazonaws.com + credentials: + s3_signing: + web_identity_credentials: + region: {{ .Values.global.region }} + bundles: + osdu/instance: + service: s3 + resource: bundle.tar.gz + {{- range .Values.partitions }} + osdu/partition/{{ . }}: + service: s3 + resource: bundle-{{ . }}.tar.gz + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/templates/deployment.yaml b/devops/aws/opa/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17a614fc18485564eef09b4e845f8ba4d94264e1 --- /dev/null +++ b/devops/aws/opa/templates/deployment.yaml @@ -0,0 +1,56 @@ +{{- template "common.deployment" (list . "opa-agent.deployment") -}} +{{- define "opa-agent.deployment" -}} +spec: + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ .Values.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: opa-bundle-server-config + mountPath: "/config" + readOnly: true + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: {{ .Values.service.readinessProbeHttpPath }} + port: http + scheme: HTTP + livenessProbe: + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: {{ .Values.service.livenessProbeHttpPath }} + port: http + scheme: HTTP + resources: + {{- toYaml .Values.resources | nindent 12 }} + args: + - run + - --ignore=.* + - --server + - -c + - /config/config + env: + {{- tpl (.Values.environmentVariables | toYaml) . | nindent 12 }} + volumes: + - name: opa-bundle-server-config + configMap: + defaultMode: 420 + name: opa-bundle-server-config +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/templates/service.yaml b/devops/aws/opa/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..892c5c032e3edf873370fdca5401897a3f284c75 --- /dev/null +++ b/devops/aws/opa/templates/service.yaml @@ -0,0 +1,9 @@ +{{- template "common.service" (list . "opa-agent.service") -}} +{{- define "opa-agent.service" -}} +spec: + ports: + - port: 80 + targetPort: {{ .Values.service.port }} + protocol: TCP + name: http +{{- end -}} diff --git a/devops/aws/opa/templates/serviceaccount.yaml b/devops/aws/opa/templates/serviceaccount.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8d90ad53de8ab3057738dc8110ef8253a222d90 --- /dev/null +++ b/devops/aws/opa/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +{{- template "common.serviceaccount" (list . "opa-agent.serviceaccount") -}} +{{- define "opa-agent.serviceaccount" -}} +## Define overrides for the service's service account resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/values.schema.json b/devops/aws/opa/values.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..14f10561e7ab0f3231b5803528bf0d69c12e73c6 --- /dev/null +++ b/devops/aws/opa/values.schema.json @@ -0,0 +1,230 @@ +{ + "$schema": "osdu-aws-services-no-ingress-helm", + "type": "object", + "default": {}, + "required": [ + "partitions", + "image", + "imagePullPolicy", + "podAnnotations", + "replicaCount", + "serviceAccountRole", + "securityContext" + ], + "properties": { + "partitions": { + "type": "array", + "default": ["osdu"], + "title": "A list of partitions", + "items": { + "type": "string", + "default": "osdu", + "title": "Partition" + }, + "examples": [ + [ + "osdu", + "opendes" + ] + ] + }, + "image": { + "type": "string", + "title": "The service container image URL" + }, + "imagePullPolicy": { + "type": "string", + "pattern": "^(Always|Never|IfNotPresent)$", + "default": "Always" + }, + "serviceParameters": { + "type": "array", + "title": "The secret parameters consumed by the service", + "items": { + "type": "object", + "required": [ + "objectName", + "objectType", + "objectAlias" + ], + "properties": { + "objectName": { + "type": "string", + "title": "The SSM or secrets parameter name", + "examples": [ + "/osdu/resource-prefix/redis/redisauthtoken", + "/osdu/resource-prefix/redis-core/end-point" + ] + }, + "objectType": { + "type": "string", + "title": "The type of parameter", + "pattern": "^(secretsmanager|ssmparameter|)$" + }, + "objectAlias": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "CACHE_CLUSTER_KEY", + "CACHE_CLUSTER_ENDPOINT" + ] + } + }, + "examples": [{ + "objectName": "/osdu/resource-prefix/redis/redisauthtoken", + "objectType": "secretsmanager", + "objectAlias": "CACHE_CLUSTER_KEY" + }, + { + "objectName": "/osdu/resource-prefix/redis-core/end-point", + "objectType": "ssmparameter", + "objectAlias": "CACHE_CLUSTER_ENDPOINT" + }] + } + }, + "environmentVariables": { + "type": "array", + "title": "The environment variables consumed by the service", + "items": { + "type": "object", + "title": "An environment variable definition", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "APPLICATION_PORT", + "AWS_REGION" + ] + }, + "value": { + "type": "string", + "title": "The value of the environment variable" + } + }, + "examples": [{ + "name": "APPLICATION_PORT", + "value": "8080" + }, + { + "name": "AWS_REGION", + "value": "us-east-1" + }] + } + }, + "podAnnotations": { + "type": "object", + "default": {} + }, + "replicaCount": { + "type": "integer", + "default": 1, + "title": "The number of pod replicas" + }, + "maxConnections": { + "type": "integer", + "default": 200, + "title": "The max number of service connections" + }, + "resources": { + "type": "object" + }, + "autoscaling": { + "type": "object" + }, + "serviceAccountRole": { + "type": "string", + "default": "", + "title": "The service account IAM role" + }, + "cors": { + "type": "object", + "default": {}, + "title": "The cors Schema", + "required": [ + "maxAge", + "allowCredentials", + "allowMethods", + "allowHeaders" + ], + "properties": { + "maxAge": { + "type": "string", + "default": "60m", + "title": "The max age of the request" + }, + "allowCredentials": { + "type": "boolean", + "default": true, + "title": "The allowCredentials Schema" + }, + "allowMethods": { + "type": "array", + "title": "A list of supported API request methods", + "items": { + "type": "string", + "title": "Supported API request methods", + "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$" + }, + "examples": [ + ["POST", + "GET", + "PATCH", + "DELETE" + ] + ] + }, + "allowHeaders": { + "type": "array", + "title": "A list of supported headers", + "items": { + "type": "string" + }, + "examples": [ + ["Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + ] + } + }, + "examples": [{ + "maxAge": "60m", + "allowCredentials": true, + "allowMethods": [ + "POST", + "GET", + "PATCH", + "DELETE" + ], + "allowHeaders": [ + "Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + }] + }, + "securityContext": { + "type": "object", + "default": {} + }, + "allowedPrincipals": { + "type": "array", + "title": "A list of allowed service principals", + "items": { + "type": "string", + "title": "Allowed principal", + "examples": [ + "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account", + "cluster.local/ns/osdu-services/sa/compliance-queue" + ] + } + } + } +} \ No newline at end of file diff --git a/devops/aws/opa/values.yaml b/devops/aws/opa/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..490d041d079674b53a37b2049c7ec5ba8be2449f --- /dev/null +++ b/devops/aws/opa/values.yaml @@ -0,0 +1,40 @@ + +# Service Config +image: openpolicyagent/opa:latest +imagePullPolicy: IfNotPresent +partitions: + - osdu + - opendes + - int-test-storage + - performance-test +service: + type: ClusterIP + port: 8181 + readinessProbeHttpPath: /health + livenessProbeHttpPath: /health + +environmentVariables: + - name: ENTITLEMENTS_BASE_URL + value: "http://os-entitlements:8080" + - name: LEGAL_BASE_URL + value: http://os-legal:8080 +podAnnotations: {} + +# Resource Config +replicaCount: 1 +resources: + limits: + memory: 200M + requests: + cpu: 500m + memory: 100M + +# Security Config +serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/{{ .Values.global.resourcePrefix }}-{{ include "common.name" . }} +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 diff --git a/devops/azure/chart/templates/deployment-opa.yaml b/devops/azure/chart/templates/deployment-opa.yaml index 64f8d16a029ddf0445aac374801b643b9d7094e1..c6f3d2a425f1946da2227020b40c858bc0894d9e 100644 --- a/devops/azure/chart/templates/deployment-opa.yaml +++ b/devops/azure/chart/templates/deployment-opa.yaml @@ -23,6 +23,10 @@ spec: containers: - name: opa image: openpolicyagent/opa:latest + lifecycle: + preStop: + exec: + command: ["sleep","90"] ports: - name: http containerPort: 8181 @@ -73,3 +77,4 @@ spec: - name: opaconfig configMap: name: opaconfig + terminationGracePeriodSeconds: 101 diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index 6e0f654d831eddbbcd0162c9ac675acff2f95a71..6869d78dc6a4fb578868e6c86f9b0ee75a30b764 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: - name: {{ .Chart.Name }} image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }} imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["sleep","90"] ports: - containerPort: 80 readinessProbe: @@ -101,4 +105,5 @@ spec: - name: USE_BUNDLES value: "yes" - name: CLOUD_PROVIDER - value: "azure" \ No newline at end of file + value: "azure" + terminationGracePeriodSeconds: 101 diff --git a/devops/azure/override-stages.yml b/devops/azure/override-stages.yml index 2a39df5a7bbdbe7b8e9fa0dbbb5ab5c14e47ffbf..a8165a10806cc4f6b15c05ec958d789f4aad7207 100644 --- a/devops/azure/override-stages.yml +++ b/devops/azure/override-stages.yml @@ -34,7 +34,7 @@ azure_containerize: only: variables: - - $AZURE == 'true' + - $AZURE == '1' azure_bootstrap: stage: bootstrap @@ -48,7 +48,7 @@ azure_bootstrap: - $AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh only: variables: - - $AZURE == 'true' + - $AZURE == '1' azure_test_py: extends: @@ -62,7 +62,7 @@ azure_test_py: - ./run-integration-tests.sh only: variables: - - $AZURE == 'true' + - $AZURE == '1' artifacts: when: on_failure paths: diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7cbc08d32a74bc23134ff55ca76e9ffe326ffe3a --- /dev/null +++ b/devops/gcp/configmap/README.md @@ -0,0 +1,65 @@ + + +# Configmap helm chart + +## Introduction + +This chart bootstraps a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**logLevel** | logging level | string | INFO | yes +**opaUrl** | OPA host | string | "http://opa" | yes +**entitlementsBaseUrl** | Entitlements host | string | "http://entitlements" | yes +**entitlementsBasePath** | Entitlements path | string | "/api/entitlements/v2/groups" | yes +**legalBaseUrl** | Legal host | string | "http://legal" | yes +**bucketName** | bucket name | string | - | yes +**useBundles** | use bundle or not | string | "yes" | yes +**cloudProvider** | Cloud provider | string | "gcp" | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**configmap** | configmap to be used | string | policy-config | yes +**appName** | name of the app | string | policy | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-policy-configmap . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-policy-configmap +``` + +[Move-to-Top](#configmap-helm-chart) diff --git a/devops/gcp/configmap/templates/policy-configmap.yaml b/devops/gcp/configmap/templates/policy-configmap.yaml index e0d103d249499c704357f3a00446f0010e9d8763..07c4cb37ecf25509b7995dc371cb678883dbacdf 100644 --- a/devops/gcp/configmap/templates/policy-configmap.yaml +++ b/devops/gcp/configmap/templates/policy-configmap.yaml @@ -2,15 +2,15 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: - LOG_LEVEL: "{{ .Values.data.log_level }}" - OPA_URL: "{{ .Values.data.opa_url }}" - ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlements_base_url }}" - ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlements_base_path }}" - LEGAL_BASE_URL: "{{ .Values.data.legal_base_url }}" - POLICY_BUCKET: "{{ .Values.data.bucket_name }}" - USE_BUNDLES: "{{ .Values.data.use_bundles }}" - CLOUD_PROVIDER: "{{ .Values.data.cloud_provider }}" + LOG_LEVEL: "{{ .Values.data.logLevel }}" + OPA_URL: "{{ .Values.data.opaUrl }}" + ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlementsBaseUrl }}" + ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlementsBasePath }}" + LEGAL_BASE_URL: "{{ .Values.data.legalBaseUrl }}" + POLICY_BUCKET: "{{ .Values.data.bucketName }}" + USE_BUNDLES: "{{ .Values.data.useBundles }}" + CLOUD_PROVIDER: "{{ .Values.data.cloudProvider }}" diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index 1b1a139ea4998dd9e71ade20ac081982589aa9b2..e1d332f701e1d15001c95cbe19094ffb010e69d6 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -1,13 +1,13 @@ data: - log_level: "INFO" - opa_url: "http://opa" - entitlements_base_url: "http://entitlements" - entitlements_base_path: "/api/entitlements/v2/groups" - legal_base_url: "http://legal" - bucket_name: "" - use_bundles: "yes" - cloud_provider: "gcp" + logLevel: "INFO" + opaUrl: "http://opa" + entitlementsBaseUrl: "http://entitlements" + entitlementsBasePath: "/api/entitlements/v2/groups" + legalBaseUrl: "http://legal" + bucketName: "" + useBundles: "yes" + cloudProvider: "gcp" conf: configmap: "policy-config" - app_name: "policy" + appName: "policy" diff --git a/devops/gcp/configmap_opa/README.md b/devops/gcp/configmap_opa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3bc71291acca0c9f27fd387adc574a46ed791034 --- /dev/null +++ b/devops/gcp/configmap_opa/README.md @@ -0,0 +1,60 @@ + + +# Configmap helm chart + +## Introduction + +This chart bootstraps a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**bucketName** | bucket name | string | - | yes +**scopes** | scope of OPA | string | "https://www.googleapis.com/auth/devstorage.read_only" | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**configmap** | configmap to be used | string | opa-config | yes +**appName** | name of the app | string | opa | yes +**dataPartitionId** | data partition id | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-opa-configmap . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-opa-configmap +``` + +[Move-to-Top](#configmap-helm-chart) diff --git a/devops/gcp/configmap_opa/templates/opa-configmap.yaml b/devops/gcp/configmap_opa/templates/opa-configmap.yaml index 1bd8b8c2e6d25ea204ddc431ca884502468ec1f7..64e061bdf09744ff05cb3ce6e57038dbd193c84d 100644 --- a/devops/gcp/configmap_opa/templates/opa-configmap.yaml +++ b/devops/gcp/configmap_opa/templates/opa-configmap.yaml @@ -2,14 +2,14 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: config.yaml: | services: gcs: - url: "https://storage.googleapis.com/storage/v1/b/{{ .Values.data.bucket_name }}/o" + url: "https://storage.googleapis.com/storage/v1/b/{{ .Values.data.bucketName }}/o" credentials: gcp_metadata: scopes: @@ -20,6 +20,6 @@ data: service: gcs # NOTE ?alt=media is required resource: 'bundle.tar.gz?alt=media' - osdu/partition/{{ .Values.conf.data_partition_id }}: + osdu/partition/{{ .Values.conf.dataPartitionId }}: service: gcs - resource: 'bundle-{{ .Values.conf.data_partition_id }}.tar.gz?alt=media' + resource: 'bundle-{{ .Values.conf.dataPartitionId }}.tar.gz?alt=media' diff --git a/devops/gcp/configmap_opa/values.yaml b/devops/gcp/configmap_opa/values.yaml index 03fb1d7edff3c57b12b887a9389212dcf5e47337..38f2233610d2563c2c57f25dfbba8ca1a6de81a7 100644 --- a/devops/gcp/configmap_opa/values.yaml +++ b/devops/gcp/configmap_opa/values.yaml @@ -1,8 +1,8 @@ data: - bucket_name: "" + bucketName: "" scopes: "https://www.googleapis.com/auth/devstorage.read_only" conf: configmap: "opa-config" - app_name: "opa" - data_partition_id: "" + appName: "opa" + dataPartitionId: "" diff --git a/devops/gcp/deploy/README.md b/devops/gcp/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bcb278b5f6f841ad1945ac8552026a6e88c2f724 --- /dev/null +++ b/devops/gcp/deploy/README.md @@ -0,0 +1,66 @@ + + +# Deploy helm chart + +## Introduction + +This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) + +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**image** | your image name | string | - | yes +**requestsCpu** | amount of requests CPU | string | 0.1 | yes +**requestsMemory** | amount of requests memory| string | 128M | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1G | yes +**serviceAccountName** | name of your service account | string | - | yes +**imagePullPolicy** | when to pull image | string | IfNotPresent | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | policy | yes +**configmap** | configmap to be used | string | policy-config | yes +**domain** | your domain | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-policy-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-policy-deploy +``` + +[Move-to-Top](#deploy-helm-chart) diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gcp/deploy/templates/deployment.yaml index df26c21df5d6416034a16fb8f0c21a01f8f4f2a9..f6b5203ff337b5c71ea978449f9e8e8587ccc654 100644 --- a/devops/gcp/deploy/templates/deployment.yaml +++ b/devops/gcp/deploy/templates/deployment.yaml @@ -2,23 +2,23 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app: "{{ .Values.conf.app_name }}" - name: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: selector: matchLabels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" replicas: 1 template: metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" annotations: rollme: {{ randAlphaNum 5 | quote }} spec: containers: - - name: "{{ .Values.conf.app_name }}" + - name: "{{ .Values.conf.appName }}" image: "{{ .Values.data.image }}" imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" envFrom: @@ -31,9 +31,9 @@ spec: - containerPort: 80 resources: requests: - cpu: "{{ .Values.data.requests_cpu }}" - memory: "{{ .Values.data.requests_memory }}" + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" limits: - cpu: "{{ .Values.data.limits_cpu }}" - memory: "{{ .Values.data.limits_memory }}" + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" serviceAccountName: "{{ .Values.data.serviceAccountName }}" diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gcp/deploy/templates/service.yaml index 878a4648aeed37daf8b79573ce6446ebea150d5c..38dba1e522b7a0af8cfd48871ac1627bcea02824 100644 --- a/devops/gcp/deploy/templates/service.yaml +++ b/devops/gcp/deploy/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" annotations: cloud.google.com/neg: '{"ingress": true}' namespace: "{{ .Release.Namespace }}" labels: - app: "{{ .Values.conf.app_name }}" - service: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + service: "{{ .Values.conf.appName }}" spec: ports: - protocol: TCP @@ -15,4 +15,4 @@ spec: targetPort: 80 name: http selector: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gcp/deploy/templates/virtual-service.yaml index 5c61613a8e52659d9e71099f9fa35d09e44637ec..4b167ea47bc8d2deba1f29ea5392f2169614ec1c 100644 --- a/devops/gcp/deploy/templates/virtual-service.yaml +++ b/devops/gcp/deploy/templates/virtual-service.yaml @@ -1,12 +1,14 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: hosts: - {{- if .Values.conf.domain }} + {{- if and .Values.conf.domain .Values.conf.onPremEnabled }} - {{ printf "osdu.%s" .Values.conf.domain | quote }} + {{- else if .Values.conf.domain }} + - {{ .Values.conf.domain | quote }} {{- else }} - "*" {{- end }} @@ -20,4 +22,4 @@ spec: - destination: port: number: 80 - host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local" + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index 9eda104f78a1c24e18472a0438d186fd212c5705..f9e1d01a3f642727755555ca54dd02b448baa6b2 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -1,13 +1,13 @@ data: - requests_cpu: "0.1" - requests_memory: "128M" - limits_cpu: "1" - limits_memory: "1G" + requestsCpu: "0.1" + requestsMemory: "128M" + limitsCpu: "1" + limitsMemory: "1G" serviceAccountName: "" imagePullPolicy: "IfNotPresent" image: "" conf: configmap: "policy-config" - app_name: "policy" + appName: "policy" domain: "" diff --git a/devops/gcp/opa/README.md b/devops/gcp/opa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2fadfd6e781709a9cae7b74974b5d18d822e1d74 --- /dev/null +++ b/devops/gcp/opa/README.md @@ -0,0 +1,66 @@ + + +# Deploy helm chart + +## Introduction + +This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) + +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**image** | your image name | string | - | yes +**requestsCpu** | amount of requests CPU | string | 0.1 | yes +**requestsMemory** | amount of requests memory| string | 128M | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1G | yes +**serviceAccountName** | name of your service account | string | opa-k8s | yes +**imagePullPolicy** | when to pull image | string | IfNotPresent | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | opa | yes +**configmap** | configmap to be used | string | opa-config | yes +**domain** | your domain | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-opa-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-opa-deploy +``` + +[Move-to-Top](#deploy-helm-chart) diff --git a/devops/gcp/opa/templates/deployment.yaml b/devops/gcp/opa/templates/deployment.yaml index 352ff305fc9367dd0246f601519e4dcfb4acb418..0ef2a89053a9a74d0b19ad4b13276eacdd68666f 100644 --- a/devops/gcp/opa/templates/deployment.yaml +++ b/devops/gcp/opa/templates/deployment.yaml @@ -1,9 +1,9 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: replicas: 1 @@ -11,15 +11,15 @@ spec: type: Recreate selector: matchLabels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" template: metadata: labels: - app: "{{ .Values.conf.app_name }}" - name: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.appName }}" spec: containers: - - name: "{{ .Values.conf.app_name }}" + - name: "{{ .Values.conf.appName }}" image: openpolicyagent/opa:latest imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" ports: @@ -31,11 +31,11 @@ spec: - "--config-file=/config/config.yaml" resources: requests: - cpu: "{{ .Values.data.requests_cpu }}" - memory: "{{ .Values.data.requests_memory }}" + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" limits: - cpu: "{{ .Values.data.limits_cpu }}" - memory: "{{ .Values.data.limits_memory }}" + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" volumeMounts: - mountPath: /config name: "{{ .Values.conf.configmap }}" diff --git a/devops/gcp/opa/templates/service.yaml b/devops/gcp/opa/templates/service.yaml index d3d3ba2db1c7e2373cb37aa3207438520ab0500c..bcd7ce28752255e0a47c6e2e5b0ae71f3e2fda04 100644 --- a/devops/gcp/opa/templates/service.yaml +++ b/devops/gcp/opa/templates/service.yaml @@ -1,9 +1,9 @@ kind: Service apiVersion: v1 metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: type: ClusterIP @@ -13,4 +13,4 @@ spec: targetPort: 8181 name: http selector: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" diff --git a/devops/gcp/opa/templates/virtual-service.yaml b/devops/gcp/opa/templates/virtual-service.yaml index c6a3807792cc0349614b727b3fa65acfc058d799..ae0b2a793e294d04ed4f9fef41c979ca07e94647 100644 --- a/devops/gcp/opa/templates/virtual-service.yaml +++ b/devops/gcp/opa/templates/virtual-service.yaml @@ -1,7 +1,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: hosts: @@ -18,6 +18,6 @@ spec: prefix: "/v1/policies" route: - destination: - host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local" + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" port: number: 8181 diff --git a/devops/gcp/opa/values.yaml b/devops/gcp/opa/values.yaml index b0e44896766cb8a5e189cb74f6e4fbc642a2564f..5a2c6ef18536e2a9dae4e4cacb601faca3f634c5 100644 --- a/devops/gcp/opa/values.yaml +++ b/devops/gcp/opa/values.yaml @@ -3,15 +3,15 @@ # Declare variables to be passed into your templates. data: - requests_cpu: "0.1" - requests_memory: "128M" - limits_cpu: "1" - limits_memory: "1G" + requestsCpu: "0.1" + requestsMemory: "128M" + limitsCpu: "1" + limitsMemory: "1G" image: "" imagePullPolicy: "IfNotPresent" serviceAccountName: "opa-k8s" conf: - app_name: "opa" + appName: "opa" configmap: "opa-config" domain: "" diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index a5d5641dfe018e0d41bb90dc47be79981c15b939..43c109d203727c23906698c0799e57074c960204 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -9,8 +9,8 @@ variables: OSDU_GCP_HELM_OPA_CONFIG_DIR: "devops/gcp/configmap_opa" # FIXME OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS: >- - --set data.bucket_name=$OSDU_GCP_POLICY_BUCKET - --set conf.data_partition_id=osdu + --set data.bucketName=$OSDU_GCP_POLICY_BUCKET + --set conf.dataPartitionId=osdu OSDU_GCP_INT_TEST_TYPE: python osdu-gcp-helm-charts-master: @@ -71,11 +71,11 @@ osdu-gcp-deploy-configmap-opa: --history-max=3 $OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-deploy-configmap-opa: @@ -95,9 +95,9 @@ osdu-gcp-dev2-deploy-configmap-opa: --set data.bucket_name=$OSDU_GCP_POLICY_BUCKET --set conf.data_partition_id=devtwo rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-deploy-opa: @@ -117,11 +117,11 @@ osdu-gcp-deploy-opa: --history-max=3 - !reference [.verify_deploy, script] rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-deploy-opa: @@ -141,9 +141,9 @@ osdu-gcp-dev2-deploy-opa: --history-max=3 - !reference [.verify_deploy, script] rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-bootstrap: @@ -162,11 +162,11 @@ osdu-gcp-bootstrap: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-bootstrap-bundle: @@ -185,11 +185,11 @@ osdu-gcp-bootstrap-bundle: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-bootstrap-bundle: @@ -208,9 +208,9 @@ osdu-gcp-dev2-bootstrap-bundle: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-dev2-bootstrap: @@ -229,9 +229,9 @@ osdu-gcp-dev2-bootstrap: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-test-python: @@ -274,7 +274,7 @@ osdu-gcp-anthos-deploy-configmap: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-deploy-deployment: @@ -283,7 +283,7 @@ osdu-gcp-anthos-deploy-deployment: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-test: @@ -292,7 +292,7 @@ osdu-gcp-anthos-test: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-test-python: @@ -301,5 +301,5 @@ osdu-gcp-anthos-test-python: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true" && $OSDU_GCP_INT_TEST_TYPE == "python"' + - if: '$OSDU_GCP == "1" && $OSDU_GCP_INT_TEST_TYPE == "python"' when: never diff --git a/devops/ibm/bootstrap-bundle.yml b/devops/ibm/bootstrap-bundle.yml index 092dc415b7dc634fed2ce10e43efd498bab3ebc3..8b76f2a1f905a2ba76c72fbcca7d92fff439370d 100644 --- a/devops/ibm/bootstrap-bundle.yml +++ b/devops/ibm/bootstrap-bundle.yml @@ -15,4 +15,4 @@ ibm_bootstrap: - ./mc cp bundle.tar.gz bundle-server/$IBM_OPA_BUNDLE_SERVER_BUCKET/ only: variables: - - $IBM == 'true' \ No newline at end of file + - $IBM == '1' diff --git a/devops/ibm/bootstrap.yml b/devops/ibm/bootstrap.yml index 7a1ace1a9148631a0e50ab334ebe83df5730ee29..cbeb90f78f4a992681da128b252c37a42dfdd7fe 100644 --- a/devops/ibm/bootstrap.yml +++ b/devops/ibm/bootstrap.yml @@ -10,10 +10,10 @@ ibm_bootstrap: - $IBM_DEPLOYMENTS_SUBDIR/bootstrap.sh only: variables: - - $IBM == 'true' + - $IBM == '1' ibm-test: needs: ["ibm_bootstrap"] only: variables: - - $IBM == 'true' + - $IBM == '1' diff --git a/frontend/admincli/Dockerfile b/frontend/admincli/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8da7b3bf33d15bb283b4d7db4e928a3c22bd8507 --- /dev/null +++ b/frontend/admincli/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9-slim +RUN useradd --create-home --shell /bin/bash app_user +WORKDIR /home/app_user +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +USER app_user +COPY . . +CMD ["bash"] diff --git a/frontend/admincli/Makefile b/frontend/admincli/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..3f6fbe281663b52ecfa2c8af2b217b128276c7b1 --- /dev/null +++ b/frontend/admincli/Makefile @@ -0,0 +1,39 @@ +IMAGE_NAME=policy-service-admincli +TAG=latest +.PHONY: build clean build_docker run test +default: build +POLICY_LOCAL_URL=http://host.docker.internal:8080 +build: + pyinstaller --onefile pol.py + +clean: + - rm -fr dist build pol.spec + - docker rm -f $(IMAGE_NAME) + +build_docker: + docker build --network host -t $(IMAGE_NAME):$(TAG) -f Dockerfile . + +run_local: + docker run -it --rm \ + -e DATA_PARTITION=${DATA_PARTITION} \ + -e TOKEN="${TOKEN}" \ + -e POLICY_URL=$(POLICY_LOCAL_URL) \ + -e ENTITLEMENTS_URL=${ENTITLEMENTS_URL} \ + -e LEGAL_URL=${LEGAL_URL} \ + --name $(IMAGE_NAME) $(IMAGE_NAME):$(TAG) + +run: + docker run -it --rm \ + -e DATA_PARTITION=${DATA_PARTITION} \ + -e TOKEN="${TOKEN}" \ + -e POLICY_URL=${POLICY_URL} \ + -e ENTITLEMENTS_URL=${ENTITLEMENTS_URL} \ + -e LEGAL_URL=${LEGAL_URL} \ + --name $(IMAGE_NAME) $(IMAGE_NAME):$(TAG) + +scan: + docker scan $(IMAGE_NAME):$(TAG) + +# requires PYTHONPATH to be set to admincli root directory +test: + pytest -v diff --git a/frontend/admincli/README.md b/frontend/admincli/README.md index 033dbb30949b16c811a18a5c1e0febcdd1ffa589..a3329ad751152ef1101cc63b86f974dc1529ea2c 100644 --- a/frontend/admincli/README.md +++ b/frontend/admincli/README.md @@ -64,4 +64,12 @@ The `ls` command supports two kinds of searching for policies: ##### Force Option: Some commands `add`, `eval` and `translate` will be default ask for comfirmation when changing data or using templating. You can bypass this by using option `--force` which is great for automation. -**The policy service team would appreciate any feedback and feature requests on the AdminCLI.** \ No newline at end of file +**The policy service team would appreciate any feedback and feature requests on the AdminCLI.** + +##### Building executables: +`make build` should build executable for your architecture in dist directory. Be sure to use python 3.9.x + +##### Building container and running container: +* `make docker_build` +* `make run` - you will still need to set the environment variables +* `make run_local` - you will still need to set the environment variables but this will override POLICY_URL to point to your local machine via docker host. diff --git a/frontend/admincli/requirements-dev.txt b/frontend/admincli/requirements-dev.txt new file mode 100644 index 0000000000000000000000000000000000000000..e53e1804d6fd15b58c608bbcbe1870475175a63e --- /dev/null +++ b/frontend/admincli/requirements-dev.txt @@ -0,0 +1,4 @@ +requests == 2.25.1 +rich == 12.5.1 +typer == 0.6.1 +pyinstaller diff --git a/frontend/admincli/tests/test.py b/frontend/admincli/tests/test_cli.py similarity index 87% rename from frontend/admincli/tests/test.py rename to frontend/admincli/tests/test_cli.py index a77444d00ea5db0e8f10f65bb0680154af8c672d..3346251ecc750f253bc2416c67fc8e308c37319c 100644 --- a/frontend/admincli/tests/test.py +++ b/frontend/admincli/tests/test_cli.py @@ -1,4 +1,7 @@ +import sys +import os from typer.testing import CliRunner +sys.path.append(os.path.abspath('..')) from pol import cli runner = CliRunner() diff --git a/postman/Bundle_based_dynamic_policies_tests.postman_collection.json b/postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json similarity index 73% rename from postman/Bundle_based_dynamic_policies_tests.postman_collection.json rename to postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json index 44c41e0be7294b5d0d3109c387ad1a752051dbcf..521ab49a41a83abac4301a313ec4c63d7f9b1d43 100644 --- a/postman/Bundle_based_dynamic_policies_tests.postman_collection.json +++ b/postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json @@ -1,23 +1,19 @@ { "info": { - "_postman_id": "58acbe53-e7fa-40c6-b37a-78a0c9295f6a", - "name": "Bundle based dynamic policies tests", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "_postman_id": "c031fd72-ff97-476d-a361-28bb25513ccc", + "name": "Bundle based dynamic policies tests M14", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "733662" }, "item": [ { - "name": "Policy service", + "name": "Policy Service M14", "item": [ { - "name": "All default policies", + "name": "All partition policies", "request": { "method": "GET", "header": [ - { - "key": "Authorization", - "value": "Bearer {{request_token}}", - "type": "text" - }, { "key": "data-partition-id", "value": "{{data_partition_id}}", @@ -25,9 +21,9 @@ } ], "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies", + "raw": "{{baseurl}}/api/policy/v1/policies", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -109,15 +105,10 @@ "response": [] }, { - "name": "Create partition policy", + "name": "Create partition policy - mytest", "request": { "method": "PUT", "header": [ - { - "key": "Authorization", - "value": "Bearer {{request_token}}", - "type": "text" - }, { "key": "data-partition-id", "value": "{{data_partition_id}}", @@ -125,13 +116,20 @@ } ], "body": { - "mode": "raw", - "raw": "package osdu.partition[\"{{data_partition_id}}\"].mytest\n\nallow = false" + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "select mytest.rego in postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/mytest.rego" + } + ] }, "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -148,7 +146,7 @@ "response": [] }, { - "name": "Created policy", + "name": "Created policy - mytest", "request": { "method": "GET", "header": [ @@ -183,7 +181,7 @@ "response": [] }, { - "name": "Delete created policy", + "name": "Delete created policy - mytest", "request": { "method": "DELETE", "header": [ @@ -199,9 +197,9 @@ } ], "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -216,6 +214,230 @@ } }, "response": [] + }, + { + "name": "Service Health", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/api/policy/v1/health", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "health" + ] + } + }, + "response": [] + }, + { + "name": "Service Info", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/api/policy/v1/info", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "info" + ] + } + }, + "response": [] + }, + { + "name": "Diagnostic about - Requires ENABLE_DEV_DIAGNOSTICS", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/diag/about", + "host": [ + "{{baseurl}}" + ], + "path": [ + "diag", + "about" + ] + } + }, + "response": [] + }, + { + "name": "Diagnostic policies - Requires ENABLE_DEV_DIAGNOSTICS Copy", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/diag/policies", + "host": [ + "{{baseurl}}" + ], + "path": [ + "diag", + "policies" + ] + } + }, + "response": [] + }, + { + "name": "Evalulate query - dataauthz", + "request": { + "method": "POST", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "eval_query1.json from postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/evaluate_query1.json" + } + ] + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/evaluations/query?policy_id=dataauthz&include_auth=true", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "evaluations", + "query" + ], + "query": [ + { + "key": "policy_id", + "value": "dataauthz" + }, + { + "key": "include_auth", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Create partition policy - search2 for translate", + "request": { + "method": "PUT", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "select search2.rego in postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/search2.rego" + } + ] + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/search2.rego", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "policies", + "osdu", + "partition", + "{{data_partition_id}}", + "search2.rego" + ] + } + }, + "response": [] + }, + { + "name": "Translate - search2", + "request": { + "method": "POST", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"query\": \"data.osdu.partition[\\\"{{data_partition_id}}\\\"].search2.allow == true\",\n \"input\": {\n \"operation\": \"view\",\n \"groups\": [\"AAA\", \"BBB\"]\n },\n \"unknowns\": [\"input.record\"]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/translate", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "translate" + ] + } + }, + "response": [] } ] }, @@ -581,6 +803,16 @@ ] } ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{access_token}}", + "type": "string" + } + ] + }, "event": [ { "listen": "prerequest", @@ -603,7 +835,7 @@ ], "variable": [ { - "key": "request_token", + "key": "access_token", "value": "" }, { diff --git a/postman/ENOPolicyServiceTesting.postman_environment.json b/postman/ENOPolicyServiceTesting.postman_environment.json index a775c3bee336c85784d57ef7581fa528b840189c..27884181b3f0ad26a24a6b59d0eaf1545f15a8b1 100644 --- a/postman/ENOPolicyServiceTesting.postman_environment.json +++ b/postman/ENOPolicyServiceTesting.postman_environment.json @@ -4,7 +4,7 @@ "values": [ { "key": "baseurl", - "value": "https://localhost:8082", + "value": "http://localhost:8080", "enabled": true }, { @@ -13,8 +13,8 @@ "enabled": true }, { - "key": "data-partition-id", - "value": "opendes", + "key": "data_partition_id", + "value": "osdu", "enabled": true }, { @@ -26,4 +26,4 @@ "_postman_variable_scope": "environment", "_postman_exported_at": "2021-03-24T19:32:49.579Z", "_postman_exported_using": "Postman/7.36.5" -} \ No newline at end of file +} diff --git a/postman/README.md b/postman/README.md index 597a19a11c8c2debea87817df4750947e0571068..8bb0a28de8118d227439cd7ea470fb5713c5b7d4 100644 --- a/postman/README.md +++ b/postman/README.md @@ -20,3 +20,4 @@ Following variables need to be set: ## Not yet supported Dynamic policies can be used for data authorization in storage and in all other services that internally use storage. The remaining gap is for the same policies to enforce data authorization in search. This is ongoing work and is expected to be completed in next release. +Bundle_based_dynamic_policies_tests_M14.postman_collection.json has been updated for M14 (fastAPI and new changes to policy service) diff --git a/postman/evaluate_query1.json b/postman/evaluate_query1.json new file mode 100644 index 0000000000000000000000000000000000000000..ee3539e4307452e91d6a5214a396bb8cc08fdb70 --- /dev/null +++ b/postman/evaluate_query1.json @@ -0,0 +1,22 @@ +{ + "input": { + "operation": "update", + "records": [ + { + "id":"osdu:test:1.4.1654807204111", + "kind":"osdu:bulkupdate:test:1.1.1654807204111", + "legal":{ + "legaltags":[ + "osdu-test-legal-tag" + ], + "otherRelevantDataCountries":["US"], + "status":"compliant" + }, + "acls":{ + "viewers":["data.default.viewers@osdu.group"], + "owners":["data.default.owners@osdu.group"] + } + } + ] + } +} diff --git a/postman/mytest.rego b/postman/mytest.rego new file mode 100644 index 0000000000000000000000000000000000000000..75294f88cad29f98417defefd8d8e997fde536eb --- /dev/null +++ b/postman/mytest.rego @@ -0,0 +1,3 @@ +package osdu.partition["osdu"].mytest + +allow = false diff --git a/postman/search2.rego b/postman/search2.rego new file mode 100644 index 0000000000000000000000000000000000000000..b8a958964755b6043026f0885795ca2d13e9763a --- /dev/null +++ b/postman/search2.rego @@ -0,0 +1,13 @@ +package osdu.partition["osdu"].search2 + +allow { + input.operation == "view" + # At least one user group needs to be in acl viewers + input.record.acl.viewers[_]==input.groups[_] +} + +allow { + input.operation == ["view", "create", "update", "delete", "purge"][_] + # At least one user group needs to be in acl owners + input.record.acl.owners[_]==input.groups[_] +}