From 8b161e97af6a5465eb1d25ab51c1a9924fb2b046 Mon Sep 17 00:00:00 2001 From: Marc Burnie Date: Sat, 28 May 2022 03:53:05 +0000 Subject: [PATCH 01/16] patching PyJWT vulnerabilities --- requirements.txt | 2 +- requirements_bootstrap.txt | 2 +- requirements_dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0274350..bacbc04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ Flask==2.0.1 requests==2.25.1 -PyJWT==2.1.0 +PyJWT==2.4.0 # To avoid "an unexpected keyword argument 'as_tuple'" error in the latest realease of 'werkzeug'-library. werkzeug==2.0.3 jinja2 diff --git a/requirements_bootstrap.txt b/requirements_bootstrap.txt index fd9ab7c..803e0e0 100644 --- a/requirements_bootstrap.txt +++ b/requirements_bootstrap.txt @@ -1,6 +1,6 @@ Flask==2.0.1 requests==2.25.1 -PyJWT==2.1.0 +PyJWT==2.4.0 # To avoid "an unexpected keyword argument 'as_tuple'" error in the latest realease of 'werkzeug'-library. werkzeug==2.0.3 jinja2 diff --git a/requirements_dev.txt b/requirements_dev.txt index 58598d1..961e385 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -8,7 +8,7 @@ httpx==0.18.1 # Note since 3.8 includes Mock 4.0+. mock==4.0.3 requests # used by starlette.TestClient for testing -PyJWT==2.1.0 +PyJWT==2.4.0 cryptography==3.4.7 # the following are used in functional integration tests -- GitLab From c2f2daa63afc499dcd1ef39a29d0e66331e87a3e Mon Sep 17 00:00:00 2001 From: Yimin Zhou Date: Wed, 29 Jun 2022 01:27:15 -0500 Subject: [PATCH 02/16] allow overwrite partition policies --- bundles/providers/azure/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/providers/azure/storage.py b/bundles/providers/azure/storage.py index d86a4d5..39c58f9 100644 --- a/bundles/providers/azure/storage.py +++ b/bundles/providers/azure/storage.py @@ -36,7 +36,7 @@ class AzureBundleStorageClient(BundleStorageClient): uri = self._get_uri(name) blob_client = self.client.get_blob_client(container=self.container_name, blob=name) #not sure how to set content type - blob_client.upload_blob(file.read(), blob_type="BlockBlob") + blob_client.upload_blob(file.read(), overwrite=True, blob_type="BlockBlob") return uri except Exception as e: logger.error(f"Failed to upload file to {uri}") -- GitLab From 423e92696aa80926799a1b9ce791dbf998e66e0a Mon Sep 17 00:00:00 2001 From: Yimin Zhou Date: Tue, 26 Jul 2022 19:42:14 -0500 Subject: [PATCH 03/16] move Helper class into Azure storage --- bundles/providers/azure/helper.py | 19 ------------------- bundles/providers/azure/storage.py | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 20 deletions(-) delete mode 100644 bundles/providers/azure/helper.py diff --git a/bundles/providers/azure/helper.py b/bundles/providers/azure/helper.py deleted file mode 100644 index 1fd2231..0000000 --- a/bundles/providers/azure/helper.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -from azure.identity import DefaultAzureCredential -from azure.keyvault.secrets import SecretClient - - -class Helper(object): - - KEY_VAULT_URI = os.getenv("KEYVAULT_URI") - - def get_key_vault_secret(self, key: str): - credential = DefaultAzureCredential() - secret_client = SecretClient( - vault_url=Helper.KEY_VAULT_URI, - credential=credential - ) - return secret_client.get_secret(key).value - - def get_storage_connection_string(self): - return self.get_key_vault_secret("airflow-storage-connection") diff --git a/bundles/providers/azure/storage.py b/bundles/providers/azure/storage.py index 39c58f9..f2c2cd0 100644 --- a/bundles/providers/azure/storage.py +++ b/bundles/providers/azure/storage.py @@ -5,7 +5,8 @@ from typing import Tuple from osdu_api.providers.types import FileLikeObject from bundles.storage import BundleStorageClient from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient -from helper import Helper +from azure.identity import DefaultAzureCredential +from azure.keyvault.secrets import SecretClient logger = logging.getLogger(__name__) @@ -40,3 +41,19 @@ class AzureBundleStorageClient(BundleStorageClient): return uri except Exception as e: logger.error(f"Failed to upload file to {uri}") + + +class Helper(object): + + KEY_VAULT_URI = os.getenv("KEYVAULT_URI") + + def get_key_vault_secret(self, key: str): + credential = DefaultAzureCredential() + secret_client = SecretClient( + vault_url=Helper.KEY_VAULT_URI, + credential=credential + ) + return secret_client.get_secret(key).value + + def get_storage_connection_string(self): + return self.get_key_vault_secret("airflow-storage-connection") -- GitLab From 01fe1a75e36bfd56851ea95eeb1c69a521d25f02 Mon Sep 17 00:00:00 2001 From: "Igor Zimovets (EPAM)" Date: Wed, 27 Jul 2022 11:33:32 +0000 Subject: [PATCH 04/16] MSCOSDU-127 update deployment --- devops/azure/chart/templates/deployment-opa.yaml | 5 +++++ devops/azure/chart/templates/deployment.yaml | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/devops/azure/chart/templates/deployment-opa.yaml b/devops/azure/chart/templates/deployment-opa.yaml index 64f8d16..c6f3d2a 100644 --- a/devops/azure/chart/templates/deployment-opa.yaml +++ b/devops/azure/chart/templates/deployment-opa.yaml @@ -23,6 +23,10 @@ spec: containers: - name: opa image: openpolicyagent/opa:latest + lifecycle: + preStop: + exec: + command: ["sleep","90"] ports: - name: http containerPort: 8181 @@ -73,3 +77,4 @@ spec: - name: opaconfig configMap: name: opaconfig + terminationGracePeriodSeconds: 101 diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index 6e0f654..6869d78 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: - name: {{ .Chart.Name }} image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }} imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["sleep","90"] ports: - containerPort: 80 readinessProbe: @@ -101,4 +105,5 @@ spec: - name: USE_BUNDLES value: "yes" - name: CLOUD_PROVIDER - value: "azure" \ No newline at end of file + value: "azure" + terminationGracePeriodSeconds: 101 -- GitLab From 07f41b2935a1098a65def9b46d245befda334cbd Mon Sep 17 00:00:00 2001 From: "Yauheni Rykhter (EPAM)" Date: Thu, 4 Aug 2022 09:56:26 +0000 Subject: [PATCH 05/16] [Policy] Refactor variables with a camel case --- devops/gcp/configmap/README.md | 65 ++++++++++++++++++ .../configmap/templates/policy-configmap.yaml | 18 ++--- devops/gcp/configmap/values.yaml | 18 ++--- devops/gcp/configmap_opa/README.md | 60 +++++++++++++++++ .../templates/opa-configmap.yaml | 8 +-- devops/gcp/configmap_opa/values.yaml | 6 +- devops/gcp/deploy/README.md | 66 +++++++++++++++++++ devops/gcp/deploy/templates/deployment.yaml | 18 ++--- devops/gcp/deploy/templates/service.yaml | 8 +-- .../gcp/deploy/templates/virtual-service.yaml | 4 +- devops/gcp/deploy/values.yaml | 10 +-- devops/gcp/opa/README.md | 66 +++++++++++++++++++ devops/gcp/opa/templates/deployment.yaml | 20 +++--- devops/gcp/opa/templates/service.yaml | 6 +- devops/gcp/opa/templates/virtual-service.yaml | 4 +- devops/gcp/opa/values.yaml | 10 +-- devops/gcp/pipeline/override-stages.yml | 4 +- 17 files changed, 324 insertions(+), 67 deletions(-) create mode 100644 devops/gcp/configmap/README.md create mode 100644 devops/gcp/configmap_opa/README.md create mode 100644 devops/gcp/deploy/README.md create mode 100644 devops/gcp/opa/README.md diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md new file mode 100644 index 0000000..7cbc08d --- /dev/null +++ b/devops/gcp/configmap/README.md @@ -0,0 +1,65 @@ + + +# Configmap helm chart + +## Introduction + +This chart bootstraps a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**logLevel** | logging level | string | INFO | yes +**opaUrl** | OPA host | string | "http://opa" | yes +**entitlementsBaseUrl** | Entitlements host | string | "http://entitlements" | yes +**entitlementsBasePath** | Entitlements path | string | "/api/entitlements/v2/groups" | yes +**legalBaseUrl** | Legal host | string | "http://legal" | yes +**bucketName** | bucket name | string | - | yes +**useBundles** | use bundle or not | string | "yes" | yes +**cloudProvider** | Cloud provider | string | "gcp" | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**configmap** | configmap to be used | string | policy-config | yes +**appName** | name of the app | string | policy | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-policy-configmap . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-policy-configmap +``` + +[Move-to-Top](#configmap-helm-chart) diff --git a/devops/gcp/configmap/templates/policy-configmap.yaml b/devops/gcp/configmap/templates/policy-configmap.yaml index e0d103d..07c4cb3 100644 --- a/devops/gcp/configmap/templates/policy-configmap.yaml +++ b/devops/gcp/configmap/templates/policy-configmap.yaml @@ -2,15 +2,15 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: - LOG_LEVEL: "{{ .Values.data.log_level }}" - OPA_URL: "{{ .Values.data.opa_url }}" - ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlements_base_url }}" - ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlements_base_path }}" - LEGAL_BASE_URL: "{{ .Values.data.legal_base_url }}" - POLICY_BUCKET: "{{ .Values.data.bucket_name }}" - USE_BUNDLES: "{{ .Values.data.use_bundles }}" - CLOUD_PROVIDER: "{{ .Values.data.cloud_provider }}" + LOG_LEVEL: "{{ .Values.data.logLevel }}" + OPA_URL: "{{ .Values.data.opaUrl }}" + ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlementsBaseUrl }}" + ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlementsBasePath }}" + LEGAL_BASE_URL: "{{ .Values.data.legalBaseUrl }}" + POLICY_BUCKET: "{{ .Values.data.bucketName }}" + USE_BUNDLES: "{{ .Values.data.useBundles }}" + CLOUD_PROVIDER: "{{ .Values.data.cloudProvider }}" diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index 1b1a139..e1d332f 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -1,13 +1,13 @@ data: - log_level: "INFO" - opa_url: "http://opa" - entitlements_base_url: "http://entitlements" - entitlements_base_path: "/api/entitlements/v2/groups" - legal_base_url: "http://legal" - bucket_name: "" - use_bundles: "yes" - cloud_provider: "gcp" + logLevel: "INFO" + opaUrl: "http://opa" + entitlementsBaseUrl: "http://entitlements" + entitlementsBasePath: "/api/entitlements/v2/groups" + legalBaseUrl: "http://legal" + bucketName: "" + useBundles: "yes" + cloudProvider: "gcp" conf: configmap: "policy-config" - app_name: "policy" + appName: "policy" diff --git a/devops/gcp/configmap_opa/README.md b/devops/gcp/configmap_opa/README.md new file mode 100644 index 0000000..3bc7129 --- /dev/null +++ b/devops/gcp/configmap_opa/README.md @@ -0,0 +1,60 @@ + + +# Configmap helm chart + +## Introduction + +This chart bootstraps a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**bucketName** | bucket name | string | - | yes +**scopes** | scope of OPA | string | "https://www.googleapis.com/auth/devstorage.read_only" | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**configmap** | configmap to be used | string | opa-config | yes +**appName** | name of the app | string | opa | yes +**dataPartitionId** | data partition id | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-opa-configmap . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-opa-configmap +``` + +[Move-to-Top](#configmap-helm-chart) diff --git a/devops/gcp/configmap_opa/templates/opa-configmap.yaml b/devops/gcp/configmap_opa/templates/opa-configmap.yaml index 1bd8b8c..64e061b 100644 --- a/devops/gcp/configmap_opa/templates/opa-configmap.yaml +++ b/devops/gcp/configmap_opa/templates/opa-configmap.yaml @@ -2,14 +2,14 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: config.yaml: | services: gcs: - url: "https://storage.googleapis.com/storage/v1/b/{{ .Values.data.bucket_name }}/o" + url: "https://storage.googleapis.com/storage/v1/b/{{ .Values.data.bucketName }}/o" credentials: gcp_metadata: scopes: @@ -20,6 +20,6 @@ data: service: gcs # NOTE ?alt=media is required resource: 'bundle.tar.gz?alt=media' - osdu/partition/{{ .Values.conf.data_partition_id }}: + osdu/partition/{{ .Values.conf.dataPartitionId }}: service: gcs - resource: 'bundle-{{ .Values.conf.data_partition_id }}.tar.gz?alt=media' + resource: 'bundle-{{ .Values.conf.dataPartitionId }}.tar.gz?alt=media' diff --git a/devops/gcp/configmap_opa/values.yaml b/devops/gcp/configmap_opa/values.yaml index 03fb1d7..38f2233 100644 --- a/devops/gcp/configmap_opa/values.yaml +++ b/devops/gcp/configmap_opa/values.yaml @@ -1,8 +1,8 @@ data: - bucket_name: "" + bucketName: "" scopes: "https://www.googleapis.com/auth/devstorage.read_only" conf: configmap: "opa-config" - app_name: "opa" - data_partition_id: "" + appName: "opa" + dataPartitionId: "" diff --git a/devops/gcp/deploy/README.md b/devops/gcp/deploy/README.md new file mode 100644 index 0000000..bcb278b --- /dev/null +++ b/devops/gcp/deploy/README.md @@ -0,0 +1,66 @@ + + +# Deploy helm chart + +## Introduction + +This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) + +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**image** | your image name | string | - | yes +**requestsCpu** | amount of requests CPU | string | 0.1 | yes +**requestsMemory** | amount of requests memory| string | 128M | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1G | yes +**serviceAccountName** | name of your service account | string | - | yes +**imagePullPolicy** | when to pull image | string | IfNotPresent | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | policy | yes +**configmap** | configmap to be used | string | policy-config | yes +**domain** | your domain | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-policy-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-policy-deploy +``` + +[Move-to-Top](#deploy-helm-chart) diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gcp/deploy/templates/deployment.yaml index df26c21..f6b5203 100644 --- a/devops/gcp/deploy/templates/deployment.yaml +++ b/devops/gcp/deploy/templates/deployment.yaml @@ -2,23 +2,23 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app: "{{ .Values.conf.app_name }}" - name: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: selector: matchLabels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" replicas: 1 template: metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" annotations: rollme: {{ randAlphaNum 5 | quote }} spec: containers: - - name: "{{ .Values.conf.app_name }}" + - name: "{{ .Values.conf.appName }}" image: "{{ .Values.data.image }}" imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" envFrom: @@ -31,9 +31,9 @@ spec: - containerPort: 80 resources: requests: - cpu: "{{ .Values.data.requests_cpu }}" - memory: "{{ .Values.data.requests_memory }}" + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" limits: - cpu: "{{ .Values.data.limits_cpu }}" - memory: "{{ .Values.data.limits_memory }}" + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" serviceAccountName: "{{ .Values.data.serviceAccountName }}" diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gcp/deploy/templates/service.yaml index 878a464..38dba1e 100644 --- a/devops/gcp/deploy/templates/service.yaml +++ b/devops/gcp/deploy/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" annotations: cloud.google.com/neg: '{"ingress": true}' namespace: "{{ .Release.Namespace }}" labels: - app: "{{ .Values.conf.app_name }}" - service: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + service: "{{ .Values.conf.appName }}" spec: ports: - protocol: TCP @@ -15,4 +15,4 @@ spec: targetPort: 80 name: http selector: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gcp/deploy/templates/virtual-service.yaml index 5c61613..5214685 100644 --- a/devops/gcp/deploy/templates/virtual-service.yaml +++ b/devops/gcp/deploy/templates/virtual-service.yaml @@ -1,7 +1,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: hosts: @@ -20,4 +20,4 @@ spec: - destination: port: number: 80 - host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local" + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index 9eda104..f9e1d01 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -1,13 +1,13 @@ data: - requests_cpu: "0.1" - requests_memory: "128M" - limits_cpu: "1" - limits_memory: "1G" + requestsCpu: "0.1" + requestsMemory: "128M" + limitsCpu: "1" + limitsMemory: "1G" serviceAccountName: "" imagePullPolicy: "IfNotPresent" image: "" conf: configmap: "policy-config" - app_name: "policy" + appName: "policy" domain: "" diff --git a/devops/gcp/opa/README.md b/devops/gcp/opa/README.md new file mode 100644 index 0000000..2fadfd6 --- /dev/null +++ b/devops/gcp/opa/README.md @@ -0,0 +1,66 @@ + + +# Deploy helm chart + +## Introduction + +This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) + +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**image** | your image name | string | - | yes +**requestsCpu** | amount of requests CPU | string | 0.1 | yes +**requestsMemory** | amount of requests memory| string | 128M | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1G | yes +**serviceAccountName** | name of your service account | string | opa-k8s | yes +**imagePullPolicy** | when to pull image | string | IfNotPresent | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | opa | yes +**configmap** | configmap to be used | string | opa-config | yes +**domain** | your domain | string | - | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-opa-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-opa-deploy +``` + +[Move-to-Top](#deploy-helm-chart) diff --git a/devops/gcp/opa/templates/deployment.yaml b/devops/gcp/opa/templates/deployment.yaml index 352ff30..0ef2a89 100644 --- a/devops/gcp/opa/templates/deployment.yaml +++ b/devops/gcp/opa/templates/deployment.yaml @@ -1,9 +1,9 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: replicas: 1 @@ -11,15 +11,15 @@ spec: type: Recreate selector: matchLabels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" template: metadata: labels: - app: "{{ .Values.conf.app_name }}" - name: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.appName }}" spec: containers: - - name: "{{ .Values.conf.app_name }}" + - name: "{{ .Values.conf.appName }}" image: openpolicyagent/opa:latest imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" ports: @@ -31,11 +31,11 @@ spec: - "--config-file=/config/config.yaml" resources: requests: - cpu: "{{ .Values.data.requests_cpu }}" - memory: "{{ .Values.data.requests_memory }}" + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" limits: - cpu: "{{ .Values.data.limits_cpu }}" - memory: "{{ .Values.data.limits_memory }}" + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" volumeMounts: - mountPath: /config name: "{{ .Values.conf.configmap }}" diff --git a/devops/gcp/opa/templates/service.yaml b/devops/gcp/opa/templates/service.yaml index d3d3ba2..bcd7ce2 100644 --- a/devops/gcp/opa/templates/service.yaml +++ b/devops/gcp/opa/templates/service.yaml @@ -1,9 +1,9 @@ kind: Service apiVersion: v1 metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: type: ClusterIP @@ -13,4 +13,4 @@ spec: targetPort: 8181 name: http selector: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" diff --git a/devops/gcp/opa/templates/virtual-service.yaml b/devops/gcp/opa/templates/virtual-service.yaml index c6a3807..ae0b2a7 100644 --- a/devops/gcp/opa/templates/virtual-service.yaml +++ b/devops/gcp/opa/templates/virtual-service.yaml @@ -1,7 +1,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: hosts: @@ -18,6 +18,6 @@ spec: prefix: "/v1/policies" route: - destination: - host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local" + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" port: number: 8181 diff --git a/devops/gcp/opa/values.yaml b/devops/gcp/opa/values.yaml index b0e4489..5a2c6ef 100644 --- a/devops/gcp/opa/values.yaml +++ b/devops/gcp/opa/values.yaml @@ -3,15 +3,15 @@ # Declare variables to be passed into your templates. data: - requests_cpu: "0.1" - requests_memory: "128M" - limits_cpu: "1" - limits_memory: "1G" + requestsCpu: "0.1" + requestsMemory: "128M" + limitsCpu: "1" + limitsMemory: "1G" image: "" imagePullPolicy: "IfNotPresent" serviceAccountName: "opa-k8s" conf: - app_name: "opa" + appName: "opa" configmap: "opa-config" domain: "" diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index a5d5641..573a4d3 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -9,8 +9,8 @@ variables: OSDU_GCP_HELM_OPA_CONFIG_DIR: "devops/gcp/configmap_opa" # FIXME OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS: >- - --set data.bucket_name=$OSDU_GCP_POLICY_BUCKET - --set conf.data_partition_id=osdu + --set data.bucketName=$OSDU_GCP_POLICY_BUCKET + --set conf.dataPartitionId=osdu OSDU_GCP_INT_TEST_TYPE: python osdu-gcp-helm-charts-master: -- GitLab From 60d06d1dfd84fde7e6e1d7859ac2736d01fa4a60 Mon Sep 17 00:00:00 2001 From: "Krisztian Molnar (EPAM)" Date: Wed, 17 Aug 2022 13:48:36 +0000 Subject: [PATCH 06/16] GONRG-5446: Fix policy pipeline --- .editorconfig | 4 ++ devops/azure/override-stages.yml | 6 +-- .../gcp/deploy/templates/virtual-service.yaml | 4 +- devops/gcp/pipeline/override-stages.yml | 48 +++++++++---------- devops/ibm/bootstrap-bundle.yml | 2 +- devops/ibm/bootstrap.yml | 4 +- 6 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5b462cc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,4 @@ +root = true + +[*] +insert_final_newline = true diff --git a/devops/azure/override-stages.yml b/devops/azure/override-stages.yml index 2a39df5..a8165a1 100644 --- a/devops/azure/override-stages.yml +++ b/devops/azure/override-stages.yml @@ -34,7 +34,7 @@ azure_containerize: only: variables: - - $AZURE == 'true' + - $AZURE == '1' azure_bootstrap: stage: bootstrap @@ -48,7 +48,7 @@ azure_bootstrap: - $AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh only: variables: - - $AZURE == 'true' + - $AZURE == '1' azure_test_py: extends: @@ -62,7 +62,7 @@ azure_test_py: - ./run-integration-tests.sh only: variables: - - $AZURE == 'true' + - $AZURE == '1' artifacts: when: on_failure paths: diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gcp/deploy/templates/virtual-service.yaml index 5214685..4b167ea 100644 --- a/devops/gcp/deploy/templates/virtual-service.yaml +++ b/devops/gcp/deploy/templates/virtual-service.yaml @@ -5,8 +5,10 @@ metadata: namespace: "{{ .Release.Namespace }}" spec: hosts: - {{- if .Values.conf.domain }} + {{- if and .Values.conf.domain .Values.conf.onPremEnabled }} - {{ printf "osdu.%s" .Values.conf.domain | quote }} + {{- else if .Values.conf.domain }} + - {{ .Values.conf.domain | quote }} {{- else }} - "*" {{- end }} diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 573a4d3..43c109d 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -71,11 +71,11 @@ osdu-gcp-deploy-configmap-opa: --history-max=3 $OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-deploy-configmap-opa: @@ -95,9 +95,9 @@ osdu-gcp-dev2-deploy-configmap-opa: --set data.bucket_name=$OSDU_GCP_POLICY_BUCKET --set conf.data_partition_id=devtwo rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-deploy-opa: @@ -117,11 +117,11 @@ osdu-gcp-deploy-opa: --history-max=3 - !reference [.verify_deploy, script] rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-deploy-opa: @@ -141,9 +141,9 @@ osdu-gcp-dev2-deploy-opa: --history-max=3 - !reference [.verify_deploy, script] rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-bootstrap: @@ -162,11 +162,11 @@ osdu-gcp-bootstrap: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-bootstrap-bundle: @@ -185,11 +185,11 @@ osdu-gcp-bootstrap-bundle: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: never - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: on_success osdu-gcp-dev2-bootstrap-bundle: @@ -208,9 +208,9 @@ osdu-gcp-dev2-bootstrap-bundle: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-dev2-bootstrap: @@ -229,9 +229,9 @@ osdu-gcp-dev2-bootstrap: - mv bundle-$DATA_PARTITION.tar.gz ./policies - gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/ rules: - - if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/' when: on_success - - if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG' + - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success osdu-gcp-test-python: @@ -274,7 +274,7 @@ osdu-gcp-anthos-deploy-configmap: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-deploy-deployment: @@ -283,7 +283,7 @@ osdu-gcp-anthos-deploy-deployment: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-test: @@ -292,7 +292,7 @@ osdu-gcp-anthos-test: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true"' + - if: '$OSDU_GCP == "1"' when: never osdu-gcp-anthos-test-python: @@ -301,5 +301,5 @@ osdu-gcp-anthos-test-python: when: never - if: "$CI_COMMIT_TAG" when: never - - if: '$OSDU_GCP == "true" && $OSDU_GCP_INT_TEST_TYPE == "python"' + - if: '$OSDU_GCP == "1" && $OSDU_GCP_INT_TEST_TYPE == "python"' when: never diff --git a/devops/ibm/bootstrap-bundle.yml b/devops/ibm/bootstrap-bundle.yml index 092dc41..8b76f2a 100644 --- a/devops/ibm/bootstrap-bundle.yml +++ b/devops/ibm/bootstrap-bundle.yml @@ -15,4 +15,4 @@ ibm_bootstrap: - ./mc cp bundle.tar.gz bundle-server/$IBM_OPA_BUNDLE_SERVER_BUCKET/ only: variables: - - $IBM == 'true' \ No newline at end of file + - $IBM == '1' diff --git a/devops/ibm/bootstrap.yml b/devops/ibm/bootstrap.yml index 7a1ace1..cbeb90f 100644 --- a/devops/ibm/bootstrap.yml +++ b/devops/ibm/bootstrap.yml @@ -10,10 +10,10 @@ ibm_bootstrap: - $IBM_DEPLOYMENTS_SUBDIR/bootstrap.sh only: variables: - - $IBM == 'true' + - $IBM == '1' ibm-test: needs: ["ibm_bootstrap"] only: variables: - - $IBM == 'true' + - $IBM == '1' -- GitLab From b0ac6eb1cd662f9140754c913ee4e5b825083baf Mon Sep 17 00:00:00 2001 From: Marc Burnie Date: Fri, 19 Aug 2022 18:10:39 +0000 Subject: [PATCH 07/16] adding AWS service helm chart --- build/aws/build-aws/buildspec.yaml | 4 + deployment/scripts/aws/AwsPolicyBootstrap.py | 4 +- deployment/scripts/aws/bootstrap.sh | 5 +- devops/aws/chart/.helmignore | 26 ++ devops/aws/chart/Chart.yaml | 12 + devops/aws/chart/README.md | 57 ++++ devops/aws/chart/templates/NOTES.txt | 4 + .../chart/templates/authorizationpolicy.yaml | 5 + devops/aws/chart/templates/deployment.yaml | 4 + .../aws/chart/templates/destinationrule.yaml | 4 + devops/aws/chart/templates/hpa.yaml | 4 + devops/aws/chart/templates/service.yaml | 4 + .../aws/chart/templates/serviceaccount.yaml | 4 + .../templates/tests/test-connection.yaml | 15 + .../aws/chart/templates/virtualservice.yaml | 4 + devops/aws/chart/values.schema.json | 271 ++++++++++++++++++ devops/aws/chart/values.yaml | 75 +++++ devops/aws/opa/.helmignore | 26 ++ devops/aws/opa/Chart.yaml | 12 + devops/aws/opa/README.md | 53 ++++ devops/aws/opa/templates/NOTES.txt | 4 + devops/aws/opa/templates/configmap.yaml | 23 ++ devops/aws/opa/templates/deployment.yaml | 56 ++++ devops/aws/opa/templates/service.yaml | 9 + devops/aws/opa/templates/serviceaccount.yaml | 4 + devops/aws/opa/values.schema.json | 230 +++++++++++++++ devops/aws/opa/values.yaml | 40 +++ 27 files changed, 956 insertions(+), 3 deletions(-) create mode 100644 devops/aws/chart/.helmignore create mode 100644 devops/aws/chart/Chart.yaml create mode 100644 devops/aws/chart/README.md create mode 100644 devops/aws/chart/templates/NOTES.txt create mode 100644 devops/aws/chart/templates/authorizationpolicy.yaml create mode 100644 devops/aws/chart/templates/deployment.yaml create mode 100644 devops/aws/chart/templates/destinationrule.yaml create mode 100644 devops/aws/chart/templates/hpa.yaml create mode 100644 devops/aws/chart/templates/service.yaml create mode 100644 devops/aws/chart/templates/serviceaccount.yaml create mode 100644 devops/aws/chart/templates/tests/test-connection.yaml create mode 100644 devops/aws/chart/templates/virtualservice.yaml create mode 100644 devops/aws/chart/values.schema.json create mode 100644 devops/aws/chart/values.yaml create mode 100644 devops/aws/opa/.helmignore create mode 100644 devops/aws/opa/Chart.yaml create mode 100644 devops/aws/opa/README.md create mode 100644 devops/aws/opa/templates/NOTES.txt create mode 100644 devops/aws/opa/templates/configmap.yaml create mode 100644 devops/aws/opa/templates/deployment.yaml create mode 100644 devops/aws/opa/templates/service.yaml create mode 100644 devops/aws/opa/templates/serviceaccount.yaml create mode 100644 devops/aws/opa/values.schema.json create mode 100644 devops/aws/opa/values.yaml diff --git a/build/aws/build-aws/buildspec.yaml b/build/aws/build-aws/buildspec.yaml index 0f90f26..32d2f03 100644 --- a/build/aws/build-aws/buildspec.yaml +++ b/build/aws/build-aws/buildspec.yaml @@ -47,6 +47,10 @@ phases: - ./tests/aws/build-aws/prepare-dist.sh - echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded - printenv + + - echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--" + - rsync -r devops/aws/* "${OUTPUT_DIR:-dist}" + - echo "Logging into Docker Hub..." - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} - echo "Building docker image..." diff --git a/deployment/scripts/aws/AwsPolicyBootstrap.py b/deployment/scripts/aws/AwsPolicyBootstrap.py index bf3dd4d..ec5b9a5 100644 --- a/deployment/scripts/aws/AwsPolicyBootstrap.py +++ b/deployment/scripts/aws/AwsPolicyBootstrap.py @@ -205,5 +205,5 @@ if __name__ == '__main__': policy_util.upsert_instance_policy_bundle(DEFAULT_POLICIES_PATH) policy_util.upsert_partition_policy_bundles( DEFAULT_POLICY_TEMPLATES_PATH) - policy_util.upsert_opa_config_map( - DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_PATH, DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_FILENAME) + # policy_util.upsert_opa_config_map( + # DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_PATH, DEFAULT_POLICY_CONFIG_MAP_TEMPLATE_FILENAME) diff --git a/deployment/scripts/aws/bootstrap.sh b/deployment/scripts/aws/bootstrap.sh index 21ba397..16dc73f 100644 --- a/deployment/scripts/aws/bootstrap.sh +++ b/deployment/scripts/aws/bootstrap.sh @@ -33,6 +33,9 @@ if [ -z "$PARTITIONS" ]; then export PARTITIONS=osdu fi +# Create and upload Policy bundles to S3 pip3 install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt - python $DEPLOYMENTS_BASE_DIR/scripts/aws/AwsPolicyBootstrap.py $AWS_REGION $AWS_SERVICE_NAMESPACE $POLICY_CONFIG_MAP_NAME $POLICY_BUCKET --partitions $PARTITIONS +# Update helm ConfigMap +# helm dependency update $DEPLOYMENTS_BASE_DIR/scripts/aws/opa-config +# helm upgrade opa-config $DEPLOYMENTS_BASE_DIR/scripts/aws/opa-config -i -n $AWS_SERVICE_NAMESPACE --set global.region=$AWS_REGION --set nameOverride=$POLICY_CONFIG_MAP_NAME --set global.policyBucket=$POLICY_BUCKET --set partitions=${PARTITIONS} --debug --dry-run \ No newline at end of file diff --git a/devops/aws/chart/.helmignore b/devops/aws/chart/.helmignore new file mode 100644 index 0000000..6e238cb --- /dev/null +++ b/devops/aws/chart/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Build files +*.html +build-info.json \ No newline at end of file diff --git a/devops/aws/chart/Chart.yaml b/devops/aws/chart/Chart.yaml new file mode 100644 index 0000000..0a0956c --- /dev/null +++ b/devops/aws/chart/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: "os-policy" +version: 0.1.0 +kubeVersion: "v1.21.x-x-x" +description: Policy Service Helm Chart for Kubernetes +type: application +appVersion: __VERSION__ +dependencies: + - name: osdu-aws-lib + version: 0.1.0 + repository: __HELM_REPO__/osdu-aws-lib/ +deprecated: false diff --git a/devops/aws/chart/README.md b/devops/aws/chart/README.md new file mode 100644 index 0000000..708ad09 --- /dev/null +++ b/devops/aws/chart/README.md @@ -0,0 +1,57 @@ +# Helm Chart + +## Introduction +The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager. + +## Prerequisites +The below software must be installed before continuing: +* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/) +* [Helm ^3.7.1](https://helm.sh/docs/intro/install/) +* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3) + +Additionally, an OSDU on AWS environment must be deployed. + +## Installation/Updating +To install or update the service application by executing the following command in the CHART folder: + +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] +``` + +To observe the Kubernetes resources before deploying them using the command: +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug +``` + +To observe the history of the current release, use the following command: +```bash +helm history [RELEASE_NAME] -n [NAMESPACE] +``` + +To revert to a previous release, use the following command: +```bash +helm rollback [RELEASE] [REVISION] -n [NAMESPACE] +``` + +### Customizing the Deployment +It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration: + +| Name | Example Value | Description | Type | Required | +| --- | ------------- | ----------- | ---- | -------- | +| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes | +| `global.resourcePrefix` | `osdu` | The resource prefix of the OSDU deployment. | str | yes | +| `global.policyBucket` | `osdu-policy-bucket` | The name of the S3 bucket containing policies. Can be found in the SSM parameter: /osdu/${resourcePrefix}/policy/s3DataBucket/name. | str | yes | +| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no | +| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no | +| `replicaCount` | `1` | The number of pod replicas to be deployed | int | no | +| `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no | +| `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no | +| `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no | + +## Uninstalling the Chart +To uninstall the helm release: + +```bash +helm uninstall [RELEASE] -n [NAMESPACE] --keep-history +``` \ No newline at end of file diff --git a/devops/aws/chart/templates/NOTES.txt b/devops/aws/chart/templates/NOTES.txt new file mode 100644 index 0000000..f15afa9 --- /dev/null +++ b/devops/aws/chart/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace +{{- if (.Values.service).apiPath }} +Application URL available at path {{ .Values.service.apiPath }} +{{- end }} \ No newline at end of file diff --git a/devops/aws/chart/templates/authorizationpolicy.yaml b/devops/aws/chart/templates/authorizationpolicy.yaml new file mode 100644 index 0000000..7194584 --- /dev/null +++ b/devops/aws/chart/templates/authorizationpolicy.yaml @@ -0,0 +1,5 @@ +{{- template "common.authzpolicy" (list . "os-policy.authzpolicy") -}} +{{- define "os-policy.authzpolicy" -}} +## Define overrides for the service authorization policy resource here +{{- end -}} + diff --git a/devops/aws/chart/templates/deployment.yaml b/devops/aws/chart/templates/deployment.yaml new file mode 100644 index 0000000..854d1b5 --- /dev/null +++ b/devops/aws/chart/templates/deployment.yaml @@ -0,0 +1,4 @@ +{{- template "common.deployment" (list . "os-policy.deployment") -}} +{{- define "os-policy.deployment" -}} +## Define overrides for the service deployment resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/destinationrule.yaml b/devops/aws/chart/templates/destinationrule.yaml new file mode 100644 index 0000000..fc5e6d3 --- /dev/null +++ b/devops/aws/chart/templates/destinationrule.yaml @@ -0,0 +1,4 @@ +{{- template "common.destinationrule" (list . "os-policy.destinationrule") -}} +{{- define "os-policy.destinationrule" -}} +## Define overrides for the service destination rule resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/hpa.yaml b/devops/aws/chart/templates/hpa.yaml new file mode 100644 index 0000000..0ad82bf --- /dev/null +++ b/devops/aws/chart/templates/hpa.yaml @@ -0,0 +1,4 @@ +{{- template "common.hpa" (list . "os-policy.hpa") -}} +{{- define "os-policy.hpa" -}} +## Define overrides for the service HPA resource here +{{- end -}} diff --git a/devops/aws/chart/templates/service.yaml b/devops/aws/chart/templates/service.yaml new file mode 100644 index 0000000..2030a3f --- /dev/null +++ b/devops/aws/chart/templates/service.yaml @@ -0,0 +1,4 @@ +{{- template "common.service" (list . "os-policy.service") -}} +{{- define "os-policy.service" -}} +## Define overrides for the service resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/serviceaccount.yaml b/devops/aws/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..9e12ece --- /dev/null +++ b/devops/aws/chart/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +{{- template "common.serviceaccount" (list . "os-policy.serviceaccount") -}} +{{- define "os-policy.serviceaccount" -}} +## Define overrides for the service's service account resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/templates/tests/test-connection.yaml b/devops/aws/chart/templates/tests/test-connection.yaml new file mode 100644 index 0000000..f341212 --- /dev/null +++ b/devops/aws/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.fullname" . }}-test-connection" + labels: + {{- include "common.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/devops/aws/chart/templates/virtualservice.yaml b/devops/aws/chart/templates/virtualservice.yaml new file mode 100644 index 0000000..5453ae0 --- /dev/null +++ b/devops/aws/chart/templates/virtualservice.yaml @@ -0,0 +1,4 @@ +{{- template "common.virtualservice" (list . "os-policy.virtualservice") -}} +{{- define "os-policy.virtualservice" -}} +## Define overrides for the service's virtual service resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/chart/values.schema.json b/devops/aws/chart/values.schema.json new file mode 100644 index 0000000..c3bc0eb --- /dev/null +++ b/devops/aws/chart/values.schema.json @@ -0,0 +1,271 @@ +{ + "$schema": "osdu-aws-services-helm", + "type": "object", + "default": {}, + "required": [ + "image", + "imagePullPolicy", + "service", + "podAnnotations", + "replicaCount", + "serviceAccountRole", + "securityContext" + ], + "properties": { + "image": { + "type": "string", + "title": "The service container image URL" + }, + "imagePullPolicy": { + "type": "string", + "pattern": "^(Always|Never|IfNotPresent)$", + "default": "Always" + }, + "service": { + "type": "object", + "required": [ + "type", + "port", + "apiPath" + ], + "properties": { + "type": { + "type": "string", + "default": "ClusterIP" + }, + "port": { + "type": "integer", + "default": 8080, + "title": "The port the service is hosted on", + "examples": [ + 8080, + 443, + 80 + ] + }, + "apiPath": { + "type": "string", + "title": "The main API path", + "examples": [ + "/api/service/v1/" + ] + }, + "readinessProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + }, + "livenessProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + }, + "startupProbeHttpPath": { + "type": "string", + "examples": [ + "/api/service/v1/actuator/health" + ] + } + }, + "examples": [{ + "type": "ClusterIP", + "port": 8080, + "apiPath": "/api/service/v1/", + "readinessProbeHttpPath": "/api/service/v1/actuator/health", + "livenessProbeHttpPath": "/api/service/v1/actuator/health", + "startupProbeHttpPath": "/api/service/v1/actuator/health" + }] + }, + "serviceParameters": { + "type": "array", + "title": "The secret parameters consumed by the service", + "items": { + "type": "object", + "required": [ + "objectName", + "objectType", + "objectAlias" + ], + "properties": { + "objectName": { + "type": "string", + "title": "The SSM or secrets parameter name", + "examples": [ + "/osdu/resource-prefix/redis/redisauthtoken", + "/osdu/resource-prefix/redis-core/end-point" + ] + }, + "objectType": { + "type": "string", + "title": "The type of parameter", + "pattern": "^(secretsmanager|ssmparameter|)$" + }, + "objectAlias": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "CACHE_CLUSTER_KEY", + "CACHE_CLUSTER_ENDPOINT" + ] + } + }, + "examples": [{ + "objectName": "/osdu/resource-prefix/redis/redisauthtoken", + "objectType": "secretsmanager", + "objectAlias": "CACHE_CLUSTER_KEY" + }, + { + "objectName": "/osdu/resource-prefix/redis-core/end-point", + "objectType": "ssmparameter", + "objectAlias": "CACHE_CLUSTER_ENDPOINT" + }] + } + }, + "environmentVariables": { + "type": "array", + "title": "The environment variables consumed by the service", + "items": { + "type": "object", + "title": "An environment variable definition", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "APPLICATION_PORT", + "AWS_REGION" + ] + }, + "value": { + "type": "string", + "title": "The value of the environment variable" + } + }, + "examples": [{ + "name": "APPLICATION_PORT", + "value": "8080" + }, + { + "name": "AWS_REGION", + "value": "us-east-1" + }] + } + }, + "podAnnotations": { + "type": "object", + "default": {} + }, + "replicaCount": { + "type": "integer", + "default": 1, + "title": "The number of pod replicas" + }, + "maxConnections": { + "type": "integer", + "default": 200, + "title": "The max number of service connections" + }, + "resources": { + "type": "object" + }, + "autoscaling": { + "type": "object" + }, + "serviceAccountRole": { + "type": "string", + "default": "", + "title": "The service account IAM role" + }, + "cors": { + "type": "object", + "default": {}, + "title": "The cors Schema", + "required": [ + "maxAge", + "allowCredentials", + "allowMethods", + "allowHeaders" + ], + "properties": { + "maxAge": { + "type": "string", + "default": "60m", + "title": "The max age of the request" + }, + "allowCredentials": { + "type": "boolean", + "default": true, + "title": "The allowCredentials Schema" + }, + "allowMethods": { + "type": "array", + "title": "A list of supported API request methods", + "items": { + "type": "string", + "title": "Supported API request methods", + "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$" + }, + "examples": [ + ["POST", + "GET", + "PATCH", + "DELETE" + ] + ] + }, + "allowHeaders": { + "type": "array", + "title": "A list of supported headers", + "items": { + "type": "string" + }, + "examples": [ + ["Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + ] + } + }, + "examples": [{ + "maxAge": "60m", + "allowCredentials": true, + "allowMethods": [ + "POST", + "GET", + "PATCH", + "DELETE" + ], + "allowHeaders": [ + "Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + }] + }, + "securityContext": { + "type": "object", + "default": {} + }, + "allowedPrincipals": { + "type": "array", + "title": "A list of allowed service principals", + "items": { + "type": "string", + "title": "Allowed principal", + "examples": [ + "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account", + "cluster.local/ns/osdu-services/sa/compliance-queue" + ] + } + } + } +} \ No newline at end of file diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml new file mode 100644 index 0000000..a54ea9c --- /dev/null +++ b/devops/aws/chart/values.yaml @@ -0,0 +1,75 @@ + +# Service Config +image: __CONTAINER__ +imagePullPolicy: IfNotPresent +service: + type: ClusterIP + port: 80 + apiPath: /api/policy/v1/ + readinessProbeHttpPath: /api/policy/v1/health + livenessProbeHttpPath: /api/policy/v1/health + +environmentVariables: + - name: APPLICATION_PORT + value: "{{ .Values.service.port }}" + - name: AWS_REGION + value: "{{ .Values.global.region }}" + - name: ENVIRONMENT + value: "{{ .Values.global.resourcePrefix }}" + - name: JAVA_OPTS + value: "-Xms538M -Xmx900M" + - name: LOG_LEVEL + value: "{{ default `INFO` .Values.global.logLevel }}" + - name: SSM_ENABLED + value: "True" + - name: SSL_ENABLED + value: "false" + - name: CLOUD_PROVIDER + value: "aws" + - name: POLICY_BUCKET + value: "{{ .Values.global.policyBucket }}" + - name: USE_BUNDLES + value: "true" + - name: ENTITLEMENTS_BASE_URL + value: "http://os-entitlements:8080" + - name: ENTITLEMENTS_BASE_PATH + value: /api/entitlements/v2/groups + - name: LEGAL_BASE_URL + value: http://os-legal:8080 + - name: OPA_URL + value: http://opa-agent +podAnnotations: {} + +# Resource Config +replicaCount: 1 +resources: + limits: + memory: 900M + requests: + cpu: 500m + memory: 300M +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Security Config +serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/{{ .Values.global.resourcePrefix }}-{{ include "common.name" . }} +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 +allowedPrincipals: + - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account + - cluster.local/ns/aws-binary-dms/sa/binary-dms + - cluster.local/ns/osdu-airflow/sa/airflow-dag-upload + - cluster.local/ns/osdu-ingest/sa/os-data-workflow + - cluster.local/ns/osdu-ingest/sa/os-ingestion-workflow + - cluster.local/ns/osdu-seismic-ddms/sa/os-seismic-store + - cluster.local/ns/osdu-well-delivery/sa/os-welldelivery + - cluster.local/ns/osdu-wellbore-ddms/sa/os-wellbore-ddms diff --git a/devops/aws/opa/.helmignore b/devops/aws/opa/.helmignore new file mode 100644 index 0000000..6e238cb --- /dev/null +++ b/devops/aws/opa/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Build files +*.html +build-info.json \ No newline at end of file diff --git a/devops/aws/opa/Chart.yaml b/devops/aws/opa/Chart.yaml new file mode 100644 index 0000000..174b501 --- /dev/null +++ b/devops/aws/opa/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: "opa-agent" +version: 0.1.0 +kubeVersion: "v1.21.x-x-x" +description: OPA Agent Helm Chart for Kubernetes +type: application +appVersion: __VERSION__ +dependencies: + - name: osdu-aws-lib + version: 0.1.0 + repository: __HELM_REPO__/osdu-aws-lib/ +deprecated: false diff --git a/devops/aws/opa/README.md b/devops/aws/opa/README.md new file mode 100644 index 0000000..0679de9 --- /dev/null +++ b/devops/aws/opa/README.md @@ -0,0 +1,53 @@ +# Helm Chart + +## Introduction +The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager. + +## Prerequisites +The below software must be installed before continuing: +* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/) +* [Helm ^3.7.1](https://helm.sh/docs/intro/install/) +* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3) + +Additionally, an OSDU on AWS environment must be deployed. + +## Installation/Updating +To install or update the service application by executing the following command in the CHART folder: + +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] +``` + +To observe the Kubernetes resources before deploying them using the command: +```bash +helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug +``` + +To observe the history of the current release, use the following command: +```bash +helm history [RELEASE_NAME] -n [NAMESPACE] +``` + +To revert to a previous release, use the following command: +```bash +helm rollback [RELEASE] [REVISION] -n [NAMESPACE] +``` + +### Customizing the Deployment +It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration: + +| Name | Example Value | Description | Type | Required | +| --- | ------------- | ----------- | ---- | -------- | +| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes | +| `global.policyBucket` | `osdu-policy-bucket` | The name of the S3 bucket containing policies. Can be found in the SSM parameter: /osdu/${resourcePrefix}/policy/s3DataBucket/name. | str | yes | +| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no | +| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no | +| `partitions` | `{osdu,opndes}` | A list of partitions with policies | array[str] | no | + +## Uninstalling the Chart +To uninstall the helm release: + +```bash +helm uninstall [RELEASE] -n [NAMESPACE] --keep-history +``` \ No newline at end of file diff --git a/devops/aws/opa/templates/NOTES.txt b/devops/aws/opa/templates/NOTES.txt new file mode 100644 index 0000000..f15afa9 --- /dev/null +++ b/devops/aws/opa/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace +{{- if (.Values.service).apiPath }} +Application URL available at path {{ .Values.service.apiPath }} +{{- end }} \ No newline at end of file diff --git a/devops/aws/opa/templates/configmap.yaml b/devops/aws/opa/templates/configmap.yaml new file mode 100644 index 0000000..0d3598e --- /dev/null +++ b/devops/aws/opa/templates/configmap.yaml @@ -0,0 +1,23 @@ +{{- template "common.configmap" (list . "opa-agent.configmap") -}} +{{- define "opa-agent.configmap" -}} +metadata: + name: opa-bundle-server-config +data: + config: | + services: + s3: + url: https://{{ .Values.global.policyBucket }}.s3.amazonaws.com + credentials: + s3_signing: + web_identity_credentials: + region: {{ .Values.global.region }} + bundles: + osdu/instance: + service: s3 + resource: bundle.tar.gz + {{- range .Values.partitions }} + osdu/partition/{{ . }}: + service: s3 + resource: bundle-{{ . }}.tar.gz + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/templates/deployment.yaml b/devops/aws/opa/templates/deployment.yaml new file mode 100644 index 0000000..17a614f --- /dev/null +++ b/devops/aws/opa/templates/deployment.yaml @@ -0,0 +1,56 @@ +{{- template "common.deployment" (list . "opa-agent.deployment") -}} +{{- define "opa-agent.deployment" -}} +spec: + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ .Values.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: opa-bundle-server-config + mountPath: "/config" + readOnly: true + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: {{ .Values.service.readinessProbeHttpPath }} + port: http + scheme: HTTP + livenessProbe: + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: {{ .Values.service.livenessProbeHttpPath }} + port: http + scheme: HTTP + resources: + {{- toYaml .Values.resources | nindent 12 }} + args: + - run + - --ignore=.* + - --server + - -c + - /config/config + env: + {{- tpl (.Values.environmentVariables | toYaml) . | nindent 12 }} + volumes: + - name: opa-bundle-server-config + configMap: + defaultMode: 420 + name: opa-bundle-server-config +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/templates/service.yaml b/devops/aws/opa/templates/service.yaml new file mode 100644 index 0000000..892c5c0 --- /dev/null +++ b/devops/aws/opa/templates/service.yaml @@ -0,0 +1,9 @@ +{{- template "common.service" (list . "opa-agent.service") -}} +{{- define "opa-agent.service" -}} +spec: + ports: + - port: 80 + targetPort: {{ .Values.service.port }} + protocol: TCP + name: http +{{- end -}} diff --git a/devops/aws/opa/templates/serviceaccount.yaml b/devops/aws/opa/templates/serviceaccount.yaml new file mode 100644 index 0000000..e8d90ad --- /dev/null +++ b/devops/aws/opa/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +{{- template "common.serviceaccount" (list . "opa-agent.serviceaccount") -}} +{{- define "opa-agent.serviceaccount" -}} +## Define overrides for the service's service account resource here +{{- end -}} \ No newline at end of file diff --git a/devops/aws/opa/values.schema.json b/devops/aws/opa/values.schema.json new file mode 100644 index 0000000..14f1056 --- /dev/null +++ b/devops/aws/opa/values.schema.json @@ -0,0 +1,230 @@ +{ + "$schema": "osdu-aws-services-no-ingress-helm", + "type": "object", + "default": {}, + "required": [ + "partitions", + "image", + "imagePullPolicy", + "podAnnotations", + "replicaCount", + "serviceAccountRole", + "securityContext" + ], + "properties": { + "partitions": { + "type": "array", + "default": ["osdu"], + "title": "A list of partitions", + "items": { + "type": "string", + "default": "osdu", + "title": "Partition" + }, + "examples": [ + [ + "osdu", + "opendes" + ] + ] + }, + "image": { + "type": "string", + "title": "The service container image URL" + }, + "imagePullPolicy": { + "type": "string", + "pattern": "^(Always|Never|IfNotPresent)$", + "default": "Always" + }, + "serviceParameters": { + "type": "array", + "title": "The secret parameters consumed by the service", + "items": { + "type": "object", + "required": [ + "objectName", + "objectType", + "objectAlias" + ], + "properties": { + "objectName": { + "type": "string", + "title": "The SSM or secrets parameter name", + "examples": [ + "/osdu/resource-prefix/redis/redisauthtoken", + "/osdu/resource-prefix/redis-core/end-point" + ] + }, + "objectType": { + "type": "string", + "title": "The type of parameter", + "pattern": "^(secretsmanager|ssmparameter|)$" + }, + "objectAlias": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "CACHE_CLUSTER_KEY", + "CACHE_CLUSTER_ENDPOINT" + ] + } + }, + "examples": [{ + "objectName": "/osdu/resource-prefix/redis/redisauthtoken", + "objectType": "secretsmanager", + "objectAlias": "CACHE_CLUSTER_KEY" + }, + { + "objectName": "/osdu/resource-prefix/redis-core/end-point", + "objectType": "ssmparameter", + "objectAlias": "CACHE_CLUSTER_ENDPOINT" + }] + } + }, + "environmentVariables": { + "type": "array", + "title": "The environment variables consumed by the service", + "items": { + "type": "object", + "title": "An environment variable definition", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "title": "The name of the environment variable consumed by the service", + "examples": [ + "APPLICATION_PORT", + "AWS_REGION" + ] + }, + "value": { + "type": "string", + "title": "The value of the environment variable" + } + }, + "examples": [{ + "name": "APPLICATION_PORT", + "value": "8080" + }, + { + "name": "AWS_REGION", + "value": "us-east-1" + }] + } + }, + "podAnnotations": { + "type": "object", + "default": {} + }, + "replicaCount": { + "type": "integer", + "default": 1, + "title": "The number of pod replicas" + }, + "maxConnections": { + "type": "integer", + "default": 200, + "title": "The max number of service connections" + }, + "resources": { + "type": "object" + }, + "autoscaling": { + "type": "object" + }, + "serviceAccountRole": { + "type": "string", + "default": "", + "title": "The service account IAM role" + }, + "cors": { + "type": "object", + "default": {}, + "title": "The cors Schema", + "required": [ + "maxAge", + "allowCredentials", + "allowMethods", + "allowHeaders" + ], + "properties": { + "maxAge": { + "type": "string", + "default": "60m", + "title": "The max age of the request" + }, + "allowCredentials": { + "type": "boolean", + "default": true, + "title": "The allowCredentials Schema" + }, + "allowMethods": { + "type": "array", + "title": "A list of supported API request methods", + "items": { + "type": "string", + "title": "Supported API request methods", + "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$" + }, + "examples": [ + ["POST", + "GET", + "PATCH", + "DELETE" + ] + ] + }, + "allowHeaders": { + "type": "array", + "title": "A list of supported headers", + "items": { + "type": "string" + }, + "examples": [ + ["Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + ] + } + }, + "examples": [{ + "maxAge": "60m", + "allowCredentials": true, + "allowMethods": [ + "POST", + "GET", + "PATCH", + "DELETE" + ], + "allowHeaders": [ + "Authorization", + "Data-Partition-Id", + "Correlation-Id", + "Content-Type" + ] + }] + }, + "securityContext": { + "type": "object", + "default": {} + }, + "allowedPrincipals": { + "type": "array", + "title": "A list of allowed service principals", + "items": { + "type": "string", + "title": "Allowed principal", + "examples": [ + "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account", + "cluster.local/ns/osdu-services/sa/compliance-queue" + ] + } + } + } +} \ No newline at end of file diff --git a/devops/aws/opa/values.yaml b/devops/aws/opa/values.yaml new file mode 100644 index 0000000..490d041 --- /dev/null +++ b/devops/aws/opa/values.yaml @@ -0,0 +1,40 @@ + +# Service Config +image: openpolicyagent/opa:latest +imagePullPolicy: IfNotPresent +partitions: + - osdu + - opendes + - int-test-storage + - performance-test +service: + type: ClusterIP + port: 8181 + readinessProbeHttpPath: /health + livenessProbeHttpPath: /health + +environmentVariables: + - name: ENTITLEMENTS_BASE_URL + value: "http://os-entitlements:8080" + - name: LEGAL_BASE_URL + value: http://os-legal:8080 +podAnnotations: {} + +# Resource Config +replicaCount: 1 +resources: + limits: + memory: 200M + requests: + cpu: 500m + memory: 100M + +# Security Config +serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/{{ .Values.global.resourcePrefix }}-{{ include "common.name" . }} +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 -- GitLab From eec77fe4043d845d0817cfc85bed27404d97fe21 Mon Sep 17 00:00:00 2001 From: "Marc Burnie [AWS]" Date: Mon, 22 Aug 2022 16:58:47 +0000 Subject: [PATCH 08/16] Update NOTICE --- NOTICE | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/NOTICE b/NOTICE index 6f5876c..b980f71 100644 --- a/NOTICE +++ b/NOTICE @@ -42,7 +42,8 @@ The following software have components provided under the terms of this license: - mock (from http://mock.readthedocs.org/en/latest/, https://github.com/testing-cabal/mock) - packaging (from https://github.com/pypa/packaging) -- pyasn1 (from http://sourceforge.net/projects/pyasn1/) +- protobuf (from https://developers.google.com/protocol-buffers/) +- pyasn1 (from http://sourceforge.net/projects/pyasn1/, https://github.com/etingof/pyasn1) - pyasn1-modules (from http://sourceforge.net/projects/pyasn1/, https://github.com/etingof/pyasn1-modules) ======================================================================== @@ -60,7 +61,7 @@ The following software have components provided under the terms of this license: - httpx (from https://github.com/encode/httpx) - idna (from https://github.com/kjd/idna) - isodate (from http://cheeseshop.python.org/pypi/isodate) -- itsdangerous (from http://github.com/mitsuhiko/itsdangerous) +- itsdangerous (from http://github.com/mitsuhiko/itsdangerous, https://palletsprojects.com/p/itsdangerous/) - jinja2 - mock (from https://github.com/testing-cabal/mock) - oauthlib (from https://github.com/idan/oauthlib, https://github.com/oauthlib/oauthlib) @@ -100,7 +101,7 @@ The following software have components provided under the terms of this license: - MarkupSafe (from https://palletsprojects.com/p/markupsafe/) - Werkzeug (from https://palletsprojects.com/p/werkzeug/) - click (from http://github.com/mitsuhiko/click) -- itsdangerous (from http://github.com/mitsuhiko/itsdangerous) +- itsdangerous (from http://github.com/mitsuhiko/itsdangerous, https://palletsprojects.com/p/itsdangerous/) - jinja2 - requests-oauthlib (from https://github.com/requests/requests-oauthlib) @@ -138,7 +139,7 @@ The following software have components provided under the terms of this license: - azure-common (from https://github.com/Azure/azure-sdk-for-python) - azure-core (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core) - azure-identity (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity, https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/identity/azure-identity) -- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets) +- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/keyvault/azure-keyvault-secrets, https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets) - azure-storage-blob (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob) - botocore (from https://github.com/boto/botocore) - cachetools (from https://github.com/tkem/cachetools/) -- GitLab From 041081bef870f2251d679e1f91afc387fc7ad7b7 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Thu, 15 Sep 2022 15:17:35 -0400 Subject: [PATCH 09/16] CI fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6556a83..1789885 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: PIP_REQUIREMENTS: "requirements.txt requirements_dev.txt app/requirements.txt" PYTHONPATH: app - PYTEST_DIR: app/tests + PYTEST_DIR: app/tests/unit AWS_BUILD_SUBDIR: build/aws/build-aws # AWS_TEST_SUBDIR: testing/storage-test-aws -- GitLab From 4c643e070fa2b4f0739849ea31edc812dde73be4 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Thu, 15 Sep 2022 16:11:26 -0400 Subject: [PATCH 10/16] Breaking pytest into unit and integration test directories. Adding support for Mocking /policies in unit testing when OPA isn't available --- app/api/policy_read_api.py | 10 ++++++++-- app/tests/integration/test_integration_020_put.py | 4 ++-- app/tests/integration/test_integration_030_get.py | 12 ++++++------ .../integration/test_integration_034_compile.py | 4 ++-- app/tests/integration/test_integration_035_eval.py | 4 ++-- app/tests/integration/test_integration_040_delete.py | 4 ++-- app/tests/integration/test_integration_045_delete.py | 4 ++-- app/tests/testlib.py | 2 +- 8 files changed, 25 insertions(+), 19 deletions(-) diff --git a/app/api/policy_read_api.py b/app/api/policy_read_api.py index c4d8dc4..fdea9a2 100644 --- a/app/api/policy_read_api.py +++ b/app/api/policy_read_api.py @@ -51,7 +51,13 @@ async def fetch_all_policies_that_match_partition(data_partition_id: Union[str, logging.info(f"data_partition_id: {data_partition_id}") opa_response = opa.fetch_opa_policies_direct() if not opa_response.ok: - raise HTTPException(status_code=opa_response.status_code, detail="Unexpected result from OPA") + cloud_provider = os.environ.get('CLOUD_PROVIDER') + if cloud_provider is None or cloud_provider == conf.MOCK: + opa_response = OpaResponse() + logger.critical(f"Error: CLOUD_PROVIDER ENV VAR not set / Mocking results for policies") + opa_response.json = {"result": {}} + else: + raise HTTPException(status_code=opa_response.status_code, detail="Unexpected result from OPA") if conf.USE_BUNDLES and 'result' in opa_response.json: logger.debug(f"OPA return {opa_response.json}") policies = opa_response.json @@ -169,4 +175,4 @@ async def fetch_partition_policy_directly_from_opa( status_code=opa_response.status_code, detail=f"Unexpected result from OPA {opa_response.message}", #headers=headers - ) \ No newline at end of file + ) diff --git a/app/tests/integration/test_integration_020_put.py b/app/tests/integration/test_integration_020_put.py index 6d644e0..3a8bb52 100644 --- a/app/tests/integration/test_integration_020_put.py +++ b/app/tests/integration/test_integration_020_put.py @@ -27,7 +27,7 @@ import testlib # override dependency injection for authentication to entitlement service from override_depends import override_require_authorized_user, set_authorize_session, ADMIN_ONLY_SVC, USER_AND_ADMIN_SVC, USER_ONLY_SVC, OTHER_ONLY_SVC -TEST_DATA_DIR = Path(__file__).resolve().parent / 'templates' +TEST_DATA_DIR = Path(__file__).resolve().parent / '../templates' client = TestClient(app) @@ -43,4 +43,4 @@ def test_put_policies_test_data_mock(token, data_partition): @pytest.mark.dependency(depends=["require_token"]) def test_put_policies_test_data_service_url(token, data_partition, bundle_pause, service_url): assert token is not None, "No token provided on command line" - testlib.put_policies_test_data(client=requests, token=token, data_partition=data_partition, service_url=service_url) \ No newline at end of file + testlib.put_policies_test_data(client=requests, token=token, data_partition=data_partition, service_url=service_url) diff --git a/app/tests/integration/test_integration_030_get.py b/app/tests/integration/test_integration_030_get.py index d9e6a8e..2692867 100644 --- a/app/tests/integration/test_integration_030_get.py +++ b/app/tests/integration/test_integration_030_get.py @@ -123,7 +123,7 @@ def test_get_diag_policies_mock(token, data_partition): #print(f"text: {r.text}",file=sys.stderr) assert r.status_code == 200, "About Page is available" p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -153,7 +153,7 @@ def test_get_diag_policies_service_url(token, data_partition, service_url): #print(f"text: {r.text}",file=sys.stderr) assert r.status_code == 200, "diag policies API is not available" p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -162,7 +162,7 @@ def test_get_diag_policies_service_url(token, data_partition, service_url): assert num_tests >= 7, "At least 7 polices were tested" @pytest.mark.dependency(depends=["require_token"]) -def test_fetch_policies(token, data_partition): +def test_fetch_policies_advanced(token, data_partition): """ get partition policies Test get /policies/osdu/partition// using built-in @@ -170,7 +170,7 @@ def test_fetch_policies(token, data_partition): """ num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -199,7 +199,7 @@ def get_osdu_partition_policies(token, data_partition, service_url=False): #pytest.skip("Skipping diagnostic test") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -224,4 +224,4 @@ def get_osdu_partition_policies(token, data_partition, service_url=False): assert f"package osdu.partition[\\\"{data_partition}\\\"].{policy_id_short}" in r.text, f"expect result for {filename}" print(r.text) num_tests = num_tests + 1 - assert num_tests >= 7, "At least 7 polices were tested" \ No newline at end of file + assert num_tests >= 7, "At least 7 polices were tested" diff --git a/app/tests/integration/test_integration_034_compile.py b/app/tests/integration/test_integration_034_compile.py index c0070ee..5c970f1 100644 --- a/app/tests/integration/test_integration_034_compile.py +++ b/app/tests/integration/test_integration_034_compile.py @@ -62,7 +62,7 @@ def test_compile(token, data_partition): num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.json' file_path = os.path.join(template_datadir, filename) @@ -79,4 +79,4 @@ def test_compile(token, data_partition): bdata = data.encode('utf-8') files = {'file': (input_json, bdata)} - post_request(token=token, data_partition=data_partition, files=files) \ No newline at end of file + post_request(token=token, data_partition=data_partition, files=files) diff --git a/app/tests/integration/test_integration_035_eval.py b/app/tests/integration/test_integration_035_eval.py index 84c01d3..799f57d 100644 --- a/app/tests/integration/test_integration_035_eval.py +++ b/app/tests/integration/test_integration_035_eval.py @@ -88,7 +88,7 @@ def test_eval_dataauthz(token, data_partition, service_url): print(f"Using Legal tag: {legal_tag}") p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.json' file_path = os.path.join(template_datadir, filename) @@ -134,4 +134,4 @@ def get_legal_tags(token, data_partition): assert x == tags, f"Expected to get {tags} from legal {retval}" legal_tags.append(tags) - return legal_tags \ No newline at end of file + return legal_tags diff --git a/app/tests/integration/test_integration_040_delete.py b/app/tests/integration/test_integration_040_delete.py index ef9465e..9ee08df 100644 --- a/app/tests/integration/test_integration_040_delete.py +++ b/app/tests/integration/test_integration_040_delete.py @@ -43,7 +43,7 @@ def test_delete_osdu_partition_policies_service_url(token, data_partition, bundl #pytest.skip("Skipping diagnostic test") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -56,4 +56,4 @@ def test_delete_osdu_partition_policies_service_url(token, data_partition, bundl assert "result" in r.text, f"result not expected {r.text}" num_tests = num_tests + 1 assert num_tests >= 7, "At least 7 polices were tested" - #time.sleep(int(bundle_pause)) \ No newline at end of file + #time.sleep(int(bundle_pause)) diff --git a/app/tests/integration/test_integration_045_delete.py b/app/tests/integration/test_integration_045_delete.py index e02ca19..e6166ac 100644 --- a/app/tests/integration/test_integration_045_delete.py +++ b/app/tests/integration/test_integration_045_delete.py @@ -43,7 +43,7 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur #pytest.skip("Skipping delete") num_tests = 0 p = os.path.dirname(os.path.abspath(__file__)) - template_datadir = os.path.join(p, 'templates') + template_datadir = os.path.join(p, '..', 'templates') for filename in sorted(os.listdir(template_datadir)): filetype='.rego' if filename.endswith(filetype): @@ -55,4 +55,4 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur assert r.status_code == 404, f"Expect 404_NOT_FOUND. Delete of {filename} {url} should already be deleted" assert "not_found" in r.text, f"expect not found message {r.text}" num_tests = num_tests + 1 - assert num_tests >= 7, "At least 7 polices were tested" \ No newline at end of file + assert num_tests >= 7, "At least 7 polices were tested" diff --git a/app/tests/testlib.py b/app/tests/testlib.py index a41c796..66006cf 100644 --- a/app/tests/testlib.py +++ b/app/tests/testlib.py @@ -67,4 +67,4 @@ def get_list_legal_tags(token, data_partition): 'data-partition-id': data_partition}) #print(json.dumps(r.json(), indent=4)) - return r.json()["legalTags"] \ No newline at end of file + return r.json()["legalTags"] -- GitLab From e8f923ebc9e759dbfe798feaf45135404820d3d0 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 10:32:50 -0400 Subject: [PATCH 11/16] Postman update --- ...olicies_tests_M14.postman_collection.json} | 286 ++++++++++++++++-- ...icyServiceTesting.postman_environment.json | 8 +- postman/README.md | 1 + postman/evaluate_query1.json | 22 ++ postman/mytest.rego | 3 + postman/search2.rego | 13 + 6 files changed, 302 insertions(+), 31 deletions(-) rename postman/{Bundle_based_dynamic_policies_tests.postman_collection.json => Bundle_based_dynamic_policies_tests_M14.postman_collection.json} (73%) create mode 100644 postman/evaluate_query1.json create mode 100644 postman/mytest.rego create mode 100644 postman/search2.rego diff --git a/postman/Bundle_based_dynamic_policies_tests.postman_collection.json b/postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json similarity index 73% rename from postman/Bundle_based_dynamic_policies_tests.postman_collection.json rename to postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json index 44c41e0..521ab49 100644 --- a/postman/Bundle_based_dynamic_policies_tests.postman_collection.json +++ b/postman/Bundle_based_dynamic_policies_tests_M14.postman_collection.json @@ -1,23 +1,19 @@ { "info": { - "_postman_id": "58acbe53-e7fa-40c6-b37a-78a0c9295f6a", - "name": "Bundle based dynamic policies tests", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "_postman_id": "c031fd72-ff97-476d-a361-28bb25513ccc", + "name": "Bundle based dynamic policies tests M14", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "733662" }, "item": [ { - "name": "Policy service", + "name": "Policy Service M14", "item": [ { - "name": "All default policies", + "name": "All partition policies", "request": { "method": "GET", "header": [ - { - "key": "Authorization", - "value": "Bearer {{request_token}}", - "type": "text" - }, { "key": "data-partition-id", "value": "{{data_partition_id}}", @@ -25,9 +21,9 @@ } ], "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies", + "raw": "{{baseurl}}/api/policy/v1/policies", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -109,15 +105,10 @@ "response": [] }, { - "name": "Create partition policy", + "name": "Create partition policy - mytest", "request": { "method": "PUT", "header": [ - { - "key": "Authorization", - "value": "Bearer {{request_token}}", - "type": "text" - }, { "key": "data-partition-id", "value": "{{data_partition_id}}", @@ -125,13 +116,20 @@ } ], "body": { - "mode": "raw", - "raw": "package osdu.partition[\"{{data_partition_id}}\"].mytest\n\nallow = false" + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "select mytest.rego in postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/mytest.rego" + } + ] }, "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -148,7 +146,7 @@ "response": [] }, { - "name": "Created policy", + "name": "Created policy - mytest", "request": { "method": "GET", "header": [ @@ -183,7 +181,7 @@ "response": [] }, { - "name": "Delete created policy", + "name": "Delete created policy - mytest", "request": { "method": "DELETE", "header": [ @@ -199,9 +197,9 @@ } ], "url": { - "raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego", "host": [ - "{{osdu_environment_url}}" + "{{baseurl}}" ], "path": [ "api", @@ -216,6 +214,230 @@ } }, "response": [] + }, + { + "name": "Service Health", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/api/policy/v1/health", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "health" + ] + } + }, + "response": [] + }, + { + "name": "Service Info", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/api/policy/v1/info", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "info" + ] + } + }, + "response": [] + }, + { + "name": "Diagnostic about - Requires ENABLE_DEV_DIAGNOSTICS", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/diag/about", + "host": [ + "{{baseurl}}" + ], + "path": [ + "diag", + "about" + ] + } + }, + "response": [] + }, + { + "name": "Diagnostic policies - Requires ENABLE_DEV_DIAGNOSTICS Copy", + "request": { + "method": "GET", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "url": { + "raw": "{{baseurl}}/diag/policies", + "host": [ + "{{baseurl}}" + ], + "path": [ + "diag", + "policies" + ] + } + }, + "response": [] + }, + { + "name": "Evalulate query - dataauthz", + "request": { + "method": "POST", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "eval_query1.json from postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/evaluate_query1.json" + } + ] + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/evaluations/query?policy_id=dataauthz&include_auth=true", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "evaluations", + "query" + ], + "query": [ + { + "key": "policy_id", + "value": "dataauthz" + }, + { + "key": "include_auth", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Create partition policy - search2 for translate", + "request": { + "method": "PUT", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "description": "select search2.rego in postman directory", + "type": "file", + "src": "/Users/shutchins/projects/osdu/merge2/policy/postman/search2.rego" + } + ] + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/search2.rego", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "policies", + "osdu", + "partition", + "{{data_partition_id}}", + "search2.rego" + ] + } + }, + "response": [] + }, + { + "name": "Translate - search2", + "request": { + "method": "POST", + "header": [ + { + "key": "data-partition-id", + "value": "{{data_partition_id}}", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"query\": \"data.osdu.partition[\\\"{{data_partition_id}}\\\"].search2.allow == true\",\n \"input\": {\n \"operation\": \"view\",\n \"groups\": [\"AAA\", \"BBB\"]\n },\n \"unknowns\": [\"input.record\"]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseurl}}/api/policy/v1/translate", + "host": [ + "{{baseurl}}" + ], + "path": [ + "api", + "policy", + "v1", + "translate" + ] + } + }, + "response": [] } ] }, @@ -581,6 +803,16 @@ ] } ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{access_token}}", + "type": "string" + } + ] + }, "event": [ { "listen": "prerequest", @@ -603,7 +835,7 @@ ], "variable": [ { - "key": "request_token", + "key": "access_token", "value": "" }, { diff --git a/postman/ENOPolicyServiceTesting.postman_environment.json b/postman/ENOPolicyServiceTesting.postman_environment.json index a775c3b..2788418 100644 --- a/postman/ENOPolicyServiceTesting.postman_environment.json +++ b/postman/ENOPolicyServiceTesting.postman_environment.json @@ -4,7 +4,7 @@ "values": [ { "key": "baseurl", - "value": "https://localhost:8082", + "value": "http://localhost:8080", "enabled": true }, { @@ -13,8 +13,8 @@ "enabled": true }, { - "key": "data-partition-id", - "value": "opendes", + "key": "data_partition_id", + "value": "osdu", "enabled": true }, { @@ -26,4 +26,4 @@ "_postman_variable_scope": "environment", "_postman_exported_at": "2021-03-24T19:32:49.579Z", "_postman_exported_using": "Postman/7.36.5" -} \ No newline at end of file +} diff --git a/postman/README.md b/postman/README.md index 597a19a..8bb0a28 100644 --- a/postman/README.md +++ b/postman/README.md @@ -20,3 +20,4 @@ Following variables need to be set: ## Not yet supported Dynamic policies can be used for data authorization in storage and in all other services that internally use storage. The remaining gap is for the same policies to enforce data authorization in search. This is ongoing work and is expected to be completed in next release. +Bundle_based_dynamic_policies_tests_M14.postman_collection.json has been updated for M14 (fastAPI and new changes to policy service) diff --git a/postman/evaluate_query1.json b/postman/evaluate_query1.json new file mode 100644 index 0000000..ee3539e --- /dev/null +++ b/postman/evaluate_query1.json @@ -0,0 +1,22 @@ +{ + "input": { + "operation": "update", + "records": [ + { + "id":"osdu:test:1.4.1654807204111", + "kind":"osdu:bulkupdate:test:1.1.1654807204111", + "legal":{ + "legaltags":[ + "osdu-test-legal-tag" + ], + "otherRelevantDataCountries":["US"], + "status":"compliant" + }, + "acls":{ + "viewers":["data.default.viewers@osdu.group"], + "owners":["data.default.owners@osdu.group"] + } + } + ] + } +} diff --git a/postman/mytest.rego b/postman/mytest.rego new file mode 100644 index 0000000..75294f8 --- /dev/null +++ b/postman/mytest.rego @@ -0,0 +1,3 @@ +package osdu.partition["osdu"].mytest + +allow = false diff --git a/postman/search2.rego b/postman/search2.rego new file mode 100644 index 0000000..b8a9589 --- /dev/null +++ b/postman/search2.rego @@ -0,0 +1,13 @@ +package osdu.partition["osdu"].search2 + +allow { + input.operation == "view" + # At least one user group needs to be in acl viewers + input.record.acl.viewers[_]==input.groups[_] +} + +allow { + input.operation == ["view", "create", "update", "delete", "purge"][_] + # At least one user group needs to be in acl owners + input.record.acl.owners[_]==input.groups[_] +} -- GitLab From 8f7a36af83b37ca83958be33ca0786711eedc291 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 11:03:12 -0400 Subject: [PATCH 12/16] updated makefile to be easier to build docker container images - added npm install --- app/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Makefile b/app/Makefile index 305de08..964cb24 100644 --- a/app/Makefile +++ b/app/Makefile @@ -26,8 +26,7 @@ build_docker: cd .. && docker build --network host -t $(IMAGE_NAME):$(TAG) -f app/Dockerfile . build_adminui: - cd ../frontend/adminui && ng build - + cd ../frontend/adminui && npm install && ng build # run policy-service locally in docker with gcp backend, assumes opa running locally run: gcp_set_token -- GitLab From b1328bceca743a3696b266bbdadec213973906d6 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 11:53:27 -0400 Subject: [PATCH 13/16] binary building and Dockerfile for admincli --- frontend/admincli/Dockerfile | 8 ++++++ frontend/admincli/Makefile | 35 ++++++++++++++++++++++++++ frontend/admincli/README.md | 10 +++++++- frontend/admincli/requirements-dev.txt | 4 +++ 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 frontend/admincli/Dockerfile create mode 100644 frontend/admincli/Makefile create mode 100644 frontend/admincli/requirements-dev.txt diff --git a/frontend/admincli/Dockerfile b/frontend/admincli/Dockerfile new file mode 100644 index 0000000..8da7b3b --- /dev/null +++ b/frontend/admincli/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9-slim +RUN useradd --create-home --shell /bin/bash app_user +WORKDIR /home/app_user +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +USER app_user +COPY . . +CMD ["bash"] diff --git a/frontend/admincli/Makefile b/frontend/admincli/Makefile new file mode 100644 index 0000000..bd70b90 --- /dev/null +++ b/frontend/admincli/Makefile @@ -0,0 +1,35 @@ +IMAGE_NAME=policy-service-admincli +TAG=latest +.PHONY: build clean build_docker run +default: build +POLICY_URL=http://host.docker.internal:8080 +build: + pyinstaller --onefile pol.py + +clean: + - rm -fr dist build pol.spec + - docker rm -f $(IMAGE_NAME) + +build_docker: + docker build --network host -t $(IMAGE_NAME):$(TAG) -f Dockerfile . + +run_local: + docker run -it --rm \ + -e DATA_PARTITION=${DATA_PARTITION} \ + -e TOKEN="${TOKEN}" \ + -e POLICY_URL=$(POLICY_URL) \ + -e ENTITLEMENTS_URL=${ENTITLEMENTS_URL} \ + -e LEGAL_URL=${LEGAL_URL} \ + --name $(IMAGE_NAME) $(IMAGE_NAME):$(TAG) + +run: + docker run -it --rm \ + -e DATA_PARTITION=${DATA_PARTITION} \ + -e TOKEN="${TOKEN}" \ + -e POLICY_URL=${POLICY_URL} \ + -e ENTITLEMENTS_URL=${ENTITLEMENTS_URL} \ + -e LEGAL_URL=${LEGAL_URL} \ + --name $(IMAGE_NAME) $(IMAGE_NAME):$(TAG) + +scan: + docker scan $(IMAGE_NAME):$(TAG) diff --git a/frontend/admincli/README.md b/frontend/admincli/README.md index 033dbb3..a3329ad 100644 --- a/frontend/admincli/README.md +++ b/frontend/admincli/README.md @@ -64,4 +64,12 @@ The `ls` command supports two kinds of searching for policies: ##### Force Option: Some commands `add`, `eval` and `translate` will be default ask for comfirmation when changing data or using templating. You can bypass this by using option `--force` which is great for automation. -**The policy service team would appreciate any feedback and feature requests on the AdminCLI.** \ No newline at end of file +**The policy service team would appreciate any feedback and feature requests on the AdminCLI.** + +##### Building executables: +`make build` should build executable for your architecture in dist directory. Be sure to use python 3.9.x + +##### Building container and running container: +* `make docker_build` +* `make run` - you will still need to set the environment variables +* `make run_local` - you will still need to set the environment variables but this will override POLICY_URL to point to your local machine via docker host. diff --git a/frontend/admincli/requirements-dev.txt b/frontend/admincli/requirements-dev.txt new file mode 100644 index 0000000..e53e180 --- /dev/null +++ b/frontend/admincli/requirements-dev.txt @@ -0,0 +1,4 @@ +requests == 2.25.1 +rich == 12.5.1 +typer == 0.6.1 +pyinstaller -- GitLab From 86dd39561373846a35e379b990a7bd16fd2f2724 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 12:25:35 -0400 Subject: [PATCH 14/16] updating tests for admincli --- frontend/admincli/Makefile | 10 +++++++--- frontend/admincli/tests/{test.py => test_cli.py} | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) rename frontend/admincli/tests/{test.py => test_cli.py} (87%) diff --git a/frontend/admincli/Makefile b/frontend/admincli/Makefile index bd70b90..3f6fbe2 100644 --- a/frontend/admincli/Makefile +++ b/frontend/admincli/Makefile @@ -1,8 +1,8 @@ IMAGE_NAME=policy-service-admincli TAG=latest -.PHONY: build clean build_docker run +.PHONY: build clean build_docker run test default: build -POLICY_URL=http://host.docker.internal:8080 +POLICY_LOCAL_URL=http://host.docker.internal:8080 build: pyinstaller --onefile pol.py @@ -17,7 +17,7 @@ run_local: docker run -it --rm \ -e DATA_PARTITION=${DATA_PARTITION} \ -e TOKEN="${TOKEN}" \ - -e POLICY_URL=$(POLICY_URL) \ + -e POLICY_URL=$(POLICY_LOCAL_URL) \ -e ENTITLEMENTS_URL=${ENTITLEMENTS_URL} \ -e LEGAL_URL=${LEGAL_URL} \ --name $(IMAGE_NAME) $(IMAGE_NAME):$(TAG) @@ -33,3 +33,7 @@ run: scan: docker scan $(IMAGE_NAME):$(TAG) + +# requires PYTHONPATH to be set to admincli root directory +test: + pytest -v diff --git a/frontend/admincli/tests/test.py b/frontend/admincli/tests/test_cli.py similarity index 87% rename from frontend/admincli/tests/test.py rename to frontend/admincli/tests/test_cli.py index a77444d..3346251 100644 --- a/frontend/admincli/tests/test.py +++ b/frontend/admincli/tests/test_cli.py @@ -1,4 +1,7 @@ +import sys +import os from typer.testing import CliRunner +sys.path.append(os.path.abspath('..')) from pol import cli runner = CliRunner() -- GitLab From 1d10c97695731ec205aca4b5630e2a96ae58e5b5 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 12:36:29 -0400 Subject: [PATCH 15/16] Documentation update --- README.md | 23 +++++++++++++++++++++++ app/README.md | 2 -- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..66a8acd --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Policy service + +Policy service is used for management and evaluation of dynamic policies in OSDU. + +## Dependencies +Policy service has a runtime dependency on Open Policy Agent [OPA](https://openpolicyagent.org/). +It also has a runtime dependency on OSDU entitlements service for API authorization. + +## Environmental variables +_ENTITLEMENTS_BASE_URL_ and _LEGAL_BASE_URL_ environmental variable must be set to run the service. The values provided should be of format _scheme://host[:port]_ and not include any part of path (e.g., https://entitlements.osdu.compay.com). + +## API authorization +To read or evaluate policies, user calling the policy service, must be a member of _service.policy.user_ or _service.policy.admin_ group. + +To create, update, or delete the policies, user must be a member of _service.policy.admin_ group. + +User can be added to these groups by using entitlements service. + +For information on how to run the policy service locally: +[Policy Service README](./app/README.md) + +There is also a Policy Service Admin CLI: +[Policy Service Admin CLI](./frontend/admincli/README.md) diff --git a/app/README.md b/app/README.md index b69cbea..6216ad3 100644 --- a/app/README.md +++ b/app/README.md @@ -126,6 +126,4 @@ For more information on [translate API](./translate/README.md). ``` ## Todo / Planned work: -* Postman collections updated * Admin UI -* A reference kubernetes helm chart/yaml will be available in future releases. -- GitLab From f916b5aa72854475e57d235c07b1b9d9ab12cdae Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Fri, 16 Sep 2022 14:04:09 -0400 Subject: [PATCH 16/16] update on port --- app/README.md | 2 ++ devops/aws/chart/values.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/README.md b/app/README.md index 6216ad3..3e5b718 100644 --- a/app/README.md +++ b/app/README.md @@ -53,6 +53,8 @@ Policy service and its integration with the rest of OSDU is an experimental feat * ttab - [programmatically open a new terminal tab](https://www.npmjs.com/package/ttab) * envsubst - [GNU gettext utilities](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) +* Port - the default for the policy service is 8080. In Kubernetes you can still expose this to port 80. + ## Testing Policy Service The [test directory](./test) contains pytest unit and integration tests and their associated data files. * Unit tests assume no services are available. diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml index a54ea9c..6bff737 100644 --- a/devops/aws/chart/values.yaml +++ b/devops/aws/chart/values.yaml @@ -4,7 +4,7 @@ image: __CONTAINER__ imagePullPolicy: IfNotPresent service: type: ClusterIP - port: 80 + port: 8080 apiPath: /api/policy/v1/ readinessProbeHttpPath: /api/policy/v1/health livenessProbeHttpPath: /api/policy/v1/health -- GitLab