diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbde3803cbcc70442abe96dfc970d8eef6f59e62..480421b3addb6d0ba5251f82d2ced16fabef2b48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,11 +65,15 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/gc-global.yml" + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/core-global.yml" + - local: "/devops/azure/override-stages.yml" - local: "/devops/aws/override-stages.yaml" - local: "/devops/aws/bootstrap.yaml" - local: "/devops/ibm/bootstrap-bundle.yml" - local: "/devops/gc/pipeline/override-stages.yml" + - local: "/devops/core-plus/pipeline/override-stages.yml" - local: "/loadtest/pipeline-loadtest.yml" - local: "/publish.yml" diff --git a/NOTICE b/NOTICE index b9c0eae13a25a204f2238f5f6be3c9a7fb50bf6a..1177b93ec0f5ae4ee8e854e25d31db02a3f8b73f 100755 --- a/NOTICE +++ b/NOTICE @@ -29,7 +29,7 @@ The following software have components provided under the terms of this license: - google-cloud-storage (from https://github.com/GoogleCloudPlatform/google-cloud-python, https://github.com/googleapis/python-storage) - google-crc32c (from https://github.com/googleapis/python-crc32c) - google-resumable-media (from https://github.com/googleapis/google-resumable-media-python) -- googleapis-common-protos (from https://github.com/googleapis/python-api-common-protos) +- googleapis-common-protos (from https://github.com/googleapis/google-cloud-python/tree/main/packages/googleapis-common-protos) - kubernetes (from https://github.com/kubernetes-client/python) - packaging (from https://pypi.org/project/packaging/, https://pypi.org/project/packaging/22.0/) - proto-plus (from https://pypi.org/project/proto-plus/) diff --git a/build/core-plus/Dockerfile b/build/core-plus/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..3be25a1610b1b0986326a38f18ed691af47b5d4a --- /dev/null +++ b/build/core-plus/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12.8-alpine + +# set environment variables +# PYTHONDONTWRITEBYTECODE - Prevents Python from writing pyc files to disc (equivalent to python -B option) +ENV PYTHONDONTWRITEBYTECODE=1 +# PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) +ENV PYTHONUNBUFFERED=1 + +RUN apk update && apk upgrade +RUN apk add gcc python3-dev musl-dev linux-headers libffi-dev openssl-dev + +EXPOSE 8080/tcp +WORKDIR /opt +COPY ./app /opt +RUN pip install --no-cache-dir --upgrade -r /opt/requirements.txt +RUN pip install setuptools==75.7.0 --upgrade + +# Create non-root user/group with numeric UID/GID +RUN addgroup -g 10001 -S nonroot && \ + adduser -u 10001 -S nonroot -G nonroot + +# Use numeric UID explicitly +USER 10001 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "4", "--proxy-headers"] diff --git a/devops/core-plus/bootstrap-osdu-module/DataPartitionBundles.py b/devops/core-plus/bootstrap-osdu-module/DataPartitionBundles.py new file mode 100644 index 0000000000000000000000000000000000000000..e4ebde0f886be4e2b8fc9071f0383c21e95030e8 --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/DataPartitionBundles.py @@ -0,0 +1,49 @@ +import os +import tarfile +from jinja2 import Environment, FileSystemLoader, select_autoescape +import argparse + + +class BootstrapDataPartitionBundles: + def create_and_upload_dp_bundles(dp_id): + tar_name = "bundle-{dp}.tar.gz".format(dp=dp_id) + dataauthz_template_name = "dataauthz_template.rego" + manifest_template_name = "manifest_template.manifest" + search_template_name = "search_template.rego" + dataauthz_filename = "dataauthz.rego" + manifest_filename = ".manifest" + search_filename = "search.rego" + template_path = "devops/core-plus/bootstrap-osdu-module/templates/" + + env = Environment( + loader=FileSystemLoader(template_path), autoescape=select_autoescape() + ) + dataauthz_template = env.get_template(dataauthz_template_name) + manifest_template = env.get_template(manifest_template_name) + search_template = env.get_template(search_template_name) + dataauthz_render = dataauthz_template.render(dp_id=dp_id) + manifest_render = manifest_template.render(dp_id=dp_id) + search_render = search_template.render(dp_id=dp_id) + + with open(dataauthz_filename, "w") as f1: + f1.write(dataauthz_render) + with open(manifest_filename, "w") as f2: + f2.write(manifest_render) + with open(search_filename, "w") as f2: + f2.write(search_render) + with tarfile.open(tar_name, "w:gz") as tar_handle: + tar_handle.add( + os.path.abspath(dataauthz_filename), arcname=dataauthz_filename + ) + tar_handle.add( + os.path.abspath(manifest_filename), arcname=manifest_filename + ) + tar_handle.add(os.path.abspath(search_filename), arcname=search_filename) + + +# Initialize class and upload bundles +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--partition", required=True, type=str) + args = parser.parse_args() + BootstrapDataPartitionBundles.create_and_upload_dp_bundles(args.partition) diff --git a/devops/core-plus/bootstrap-osdu-module/Dockerfile b/devops/core-plus/bootstrap-osdu-module/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1415f0460c469b460c7718f16976bd81bba1d37c --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/Dockerfile @@ -0,0 +1,19 @@ +FROM gcr.io/google.com/cloudsdktool/cloud-sdk:alpine + +WORKDIR /opt +ENV PIP_BREAK_SYSTEM_PACKAGES 1 +COPY ./requirements_bootstrap.txt ./devops/core-plus/bootstrap-osdu-module/*.sh /opt/ +COPY ./deployment/ /opt/deployment +COPY ./devops/core-plus/bootstrap-osdu-module /opt/devops/core-plus/bootstrap-osdu-module + +RUN chmod 775 /opt/bootstrap_policy.sh +RUN apk add py3-pip wget jq +RUN pip3 install -r /opt/requirements_bootstrap.txt -r /opt/devops/core-plus/bootstrap-osdu-module/requirements.txt +RUN wget --quiet https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc && mv mc /usr/bin/mc + +RUN addgroup -g 10001 -S nonroot \ + && adduser -h /opt -G nonroot -S -u 10001 nonroot +RUN chown -R 10001:10001 /opt +USER 10001:10001 + +CMD ["/bin/bash", "-c", "/opt/bootstrap_policy.sh && sleep 365d"] diff --git a/devops/core-plus/bootstrap-osdu-module/README.md b/devops/core-plus/bootstrap-osdu-module/README.md new file mode 100644 index 0000000000000000000000000000000000000000..adfaf52c3a6629252c88349e3df35cdec55415db --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/README.md @@ -0,0 +1,5 @@ +# Prerequisites + +* create workload identity gke service account (it is mandatory that bootstrap script will be running from SA) +* kubernetes job should be create in namespace that is free from istio-injection (job will be in RUNNING state indefenetly with side-car container) +* set all required ENV variables, they are listed in TF bootstrap job.tf under env directive diff --git a/devops/core-plus/bootstrap-osdu-module/bootstrap_policy.sh b/devops/core-plus/bootstrap-osdu-module/bootstrap_policy.sh new file mode 100644 index 0000000000000000000000000000000000000000..8876e5a73f1aaa81f839782009adb9e981494461 --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/bootstrap_policy.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# The following script renders and archives bundles of policies for instance and partition level +# After that archives are uploaded to GCS bucket or MinIO bucket + +set -ex + +source ./validate-env.sh "PARTITION_BASE_URL" + +create_instance_bundles() { + # Renders and archives intance level policies + echo "Archiving bundle of instance policies..." + tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose + mkdir --parents /opt/policies + mv bundle.tar.gz "$_" + echo "Instance policies archive is ready" +} + +create_partition_bundle() { + # Renders and archives policies for data_partition + # Creates archive named bundle-<data_partition>.tar.gz in /opt/policies + # Args: $1 - data_partition_id + + DATA_PARTITION=$1 + echo "Archiving bundle of policies for parition: ${DATA_PARTITION}..." + python3 /opt/devops/core-plus/bootstrap-osdu-module/DataPartitionBundles.py --partition "${DATA_PARTITION}" + mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies + echo "${DATA_PARTITION} partition archive is ready" +} + +bootstrap_minio() { + echo "Configuring mc tool" + mc alias set minio "${MINIO_HOST}":"${MINIO_PORT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + echo "Pushing archives to Minio bucket" + for file in /opt/policies/*; do + echo "Processing $file:" + file_name=${file##*/} + # Check if file already exists + if mc stat minio/"${POLICY_BUCKET}"/"$file_name" >/dev/null 2>&1; then + echo "Skipping $file: already exists in bucket" + else + mc cp "$file" minio/"${POLICY_BUCKET}"/"$file_name" + fi + done + echo "Bootstrap finished successfully" +} + +# Main part +source ./validate-env.sh "POLICY_BUCKET" + +# Creating instance bundles +create_instance_bundles + +# Get all partitions +PARTITIONS_LIST=$(curl --location "${PARTITION_BASE_URL}/api/partition/v1/partitions" | jq -r '[.[] | select(. != "system")] | join(",")') + +# Check for partition bootstrap +if [ -z "$PARTITIONS_LIST" ] + then + echo "Partition bootstrap is not finished" + exit 1 +fi + +IFS=',' read -ra PARTITIONS <<< "${PARTITIONS_LIST}" +echo $PARTITIONS + +# Creating partition bundles +for PARTITION in "${PARTITIONS[@]}"; do + create_partition_bundle "${PARTITION}" +done + +# Uploading bundles to gcs/minio bucket + source ./validate-env.sh "MINIO_HOST" + source ./validate-env.sh "MINIO_ACCESS_KEY" + source ./validate-env.sh "MINIO_SECRET_KEY" + source ./validate-env.sh "MINIO_PORT" + bootstrap_minio + +touch /tmp/bootstrap_ready diff --git a/devops/core-plus/bootstrap-osdu-module/requirements.txt b/devops/core-plus/bootstrap-osdu-module/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a8f7292e0382579f9d1b4c8bbddd28698e46ddc2 --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/requirements.txt @@ -0,0 +1 @@ +jinja2==3.1.2 diff --git a/devops/core-plus/bootstrap-osdu-module/templates/dataauthz_template.rego b/devops/core-plus/bootstrap-osdu-module/templates/dataauthz_template.rego new file mode 100644 index 0000000000000000000000000000000000000000..7372a4e66790ce89c8410216bd7cd661c0850f58 --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/templates/dataauthz_template.rego @@ -0,0 +1,5 @@ +package osdu.partition["{{dp_id}}"].dataauthz + +import data.osdu.instance.dataauthz as centralauthz + +records := centralauthz.records diff --git a/devops/core-plus/bootstrap-osdu-module/templates/manifest_template.manifest b/devops/core-plus/bootstrap-osdu-module/templates/manifest_template.manifest new file mode 100644 index 0000000000000000000000000000000000000000..4a834c2a3ed30987b6541d8b9a681d53d2881dee --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/templates/manifest_template.manifest @@ -0,0 +1,3 @@ +{ + "roots": ["osdu/partition/{{dp_id}}"] +} diff --git a/devops/core-plus/bootstrap-osdu-module/templates/search_template.rego b/devops/core-plus/bootstrap-osdu-module/templates/search_template.rego new file mode 100644 index 0000000000000000000000000000000000000000..9f394427a56aee0139025df15930ca42b6ec39e5 --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/templates/search_template.rego @@ -0,0 +1,15 @@ +package osdu.partition["{{dp_id}}"].search + +default allow = false + +allow = true { + input.operation == "view" + # At least one user group needs to be in acl viewers + input.record.acl.viewers[_]==input.groups[_] +} + +allow = true { + input.operation == ["view", "create", "update", "delete", "purge"][_] + # At least one user group needs to be in acl owners + input.record.acl.owners[_]==input.groups[_] +} diff --git a/devops/core-plus/bootstrap-osdu-module/validate-env.sh b/devops/core-plus/bootstrap-osdu-module/validate-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..9de17ff4a83b4e297bba6aec875088dd313c416b --- /dev/null +++ b/devops/core-plus/bootstrap-osdu-module/validate-env.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +{ set +x ;} 2> /dev/null # disable output to prevent secret logging +set -e + +ENV_VAR_NAME=$1 + +if [ "${!ENV_VAR_NAME}" = "" ] +then + echo "Missing environment variable '$ENV_VAR_NAME'. Please provide all variables and try again" + { set -x ;} 2> /dev/null # enable output back + exit 1 +fi + +{ set -x ;} 2> /dev/null # enable output back diff --git a/devops/core-plus/deploy/Chart.yaml b/devops/core-plus/deploy/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..709e8f74ae00e55e04ed99b34ffb4168ac192a95 --- /dev/null +++ b/devops/core-plus/deploy/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: core-plus-policy-deploy +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.19.0" diff --git a/devops/core-plus/deploy/README.md b/devops/core-plus/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..52c64e7f98918b758ebe36c8403e42d02ef9d679 --- /dev/null +++ b/devops/core-plus/deploy/README.md @@ -0,0 +1,128 @@ +<!--- Deploy --> + +# CORE Policy service + +## Introduction + +This chart deploys policy service on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.23.12) with **Istio** (1.15) + +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +- **Kubectl** (version: v1.23.12 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Global variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes +**global.limitsEnabled** | whether CPU and memory limits are enabled | boolean | `true` | yes +**global.dataPartitionId** | data partition id | string | - | yes +**global.logLevel** | severity of logging level | string | `ERROR` | yes + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|----------| +**data.logLevel** | logging severity level for this service only | string | - | yes, only if differs from the `global.logLevel` +**data.image** | policy image name | string | - | yes +**data.requestsCpu** | amount of requests CPU | string | `10m` | yes +**data.requestsMemory** | amount of requests memory| string | `200Mi` | yes +**data.limitsCpu** | CPU limit | string | `1` | only if `global.limitsEnabled` is true +**data.limitsMemory** | memory limit | string | `1G` | only if `global.limitsEnabled` is true +**data.serviceAccountName** | name of your service account | string | - | yes +**data.imagePullPolicy** | when to pull image | string | `IfNotPresent` | yes +**data.bucketName** | bucket name | string | - | yes +**data.scopes** | scope of OPA | string | `https://www.googleapis.com/auth/devstorage.read_only` | yes +**data.entitlementsHost** | Entitlements host | string | `http://entitlements` | yes +**data.entitlementsBasePath** | Entitlements path | string | `/api/entitlements/v2/groups` | yes +**data.useBundles** | use bundle or not | string | `yes` | yes +**data.legalHost** | Legal host | string | `http://legal` | yes +**data.partitionHost** | Partition host | string | `http://partition` | yes + +### Baremetal variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**data.minioHost** | minio host | string | `http://minio:9000` | yes +**conf.minioSecretName** | secret name for the app | string | `policy-minio-secret` | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**conf.appName** | name of the app | string | `policy` | yes +**conf.configmap** | configmap to be used | string | `policy-config` | yes +**conf.bootstrapSecretName** | secret name for the bootstrap | string | `minio-bootstrap-secret` | yes +**conf.minDelaySeconds** | min delay for bundle download | num | `6` | yes +**conf.maxDelaySeconds** | max delay for bundle download | num | `12` | yes + +### Bootstrap variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**data.bootstrapImage** | image for bootstrap deployment | string | - | yes +**data.bootstrapServiceAccountName** | service account for bootstrap deployment | string | - | yes + +### OPA variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**opa.conf.configmap** | configmap to be used | string | `opa-config` | yes +**opa.conf.envConfig** | configmap with env vars | string | `opa-env-config` | yes +**opa.conf.appName** | name of the app | string | `opa` | yes +**opa.data.serviceAccountName** | name of your service account | string | `opa-k8s` | yes +**opa.data.image** | image name | string | - | yes + +### ISTIO variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**istio.proxyCPU** | CPU request for Envoy sidecars | string | `10m` | yes +**istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | `500m` | yes +**istio.proxyMemory** | memory request for Envoy sidecars | string | `100Mi` | yes +**istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | `512Mi` | yes +**istio.bootstrapProxyCPU** | CPU request for Envoy sidecars | string | `10m` | yes +**istio.bootstrapProxyCPULimit** | CPU limit for Envoy sidecars | string | `100m` | yes + +### Methodology for Parameter Calculation variables: **hpa.targetValue**, **limits.maxTokens** and **limits.tokensPerFill** + +The parameters **hpa.targetValue**, **limits.maxTokens** and **limits.tokensPerFill** were determined through empirical testing during load testing. These tests were conducted using the N2D machine series, which can run on either AMD EPYC Milan or AMD EPYC Rome processors. The values were fine-tuned to ensure optimal performance under typical workloads. + +### Recommendations for New Instance Types + +When changing the instance type to a newer generation, such as the C3D series, it is essential to conduct new load testing. This ensures the parameters are recalibrated to match the performance characteristics of the new processor architecture, optimizing resource utilization and maintaining application stability. + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install core-plus-policy-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall core-plus-policy-deploy +``` + +[Move-to-Top](#core-plus-policy-service) diff --git a/devops/core-plus/deploy/templates/opa-deployment.yaml b/devops/core-plus/deploy/templates/opa-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e1f4d131be9065691b0bf9f8463fa0bc866ca1d --- /dev/null +++ b/devops/core-plus/deploy/templates/opa-deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ .Values.opa.conf.appName }}" + labels: + app: "{{ .Values.opa.conf.appName }}" + type: core + namespace: "{{ .Release.Namespace }}" +spec: + replicas: {{ .Values.opa.conf.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + app: "{{ .Values.opa.conf.appName }}" + template: + metadata: + labels: + app: "{{ .Values.opa.conf.appName }}" + annotations: + rollme: {{ randAlphaNum 5 | quote }} + sidecar.istio.io/proxyCPU: {{ .Values.istio.proxyCPU | quote }} + sidecar.istio.io/proxyMemory: {{ .Values.istio.proxyMemory | quote }} + sidecar.istio.io/proxyCPULimit: {{ .Values.istio.proxyCPULimit | quote }} + sidecar.istio.io/proxyMemoryLimit: {{ .Values.istio.proxyMemoryLimit | quote }} + name: "{{ .Values.opa.conf.appName }}" + spec: + initContainers: + - name: "{{ .Values.opa.init.name }}" + image: "{{ .Values.opa.init.image }}" + securityContext: + runAsUser: 1337 + volumeMounts: + - name: config + mountPath: /config + envFrom: + - configMapRef: + name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }} + containers: + - name: "{{ .Values.opa.conf.appName }}" + image: "{{ .Values.opa.data.image }}" + imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" + ports: + - containerPort: 8181 + args: + - "run" + - "--ignore=.*" # exclude hidden dirs created by Kubernetes + - "--server" + - "--config-file=/config/config.yaml" + resources: + requests: + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.opa.data.requestsMemory }}" + {{- if .Values.global.limitsEnabled }} + limits: + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" + {{- end }} + volumeMounts: + - name: config + mountPath: /config + envFrom: + - configMapRef: + name: "{{ .Values.opa.conf.envConfig }}" + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.conf.minioSecretName }}" + key: MINIO_ACCESS_KEY + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.conf.minioSecretName }}" + key: MINIO_SECRET_KEY + - name: AWS_REGION + valueFrom: + secretKeyRef: + name: "{{ .Values.conf.minioSecretName }}" + key: AWS_REGION + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + volumes: + - name: config + emptyDir: {} + serviceAccountName: "{{ .Values.opa.data.serviceAccountName }}" diff --git a/devops/core-plus/deploy/templates/opa-env-configmap.yaml b/devops/core-plus/deploy/templates/opa-env-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e00acf413db948017773d8eff0ff14d3ca57e59 --- /dev/null +++ b/devops/core-plus/deploy/templates/opa-env-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: "{{ .Values.opa.conf.appName }}" + name: "{{ .Values.opa.conf.envConfig }}" + namespace: "{{ .Release.Namespace }}" +data: + LEGAL_BASE_URL: "{{ .Values.data.legalHost }}" + ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlementsHost }}" diff --git a/devops/core-plus/deploy/templates/opa-service-account.yaml b/devops/core-plus/deploy/templates/opa-service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a0be4d4adc227224562e06182d3bbe654ced3d6 --- /dev/null +++ b/devops/core-plus/deploy/templates/opa-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Values.opa.data.serviceAccountName }}" + namespace: "{{ .Release.Namespace }}" diff --git a/devops/core-plus/deploy/templates/opa-service.yaml b/devops/core-plus/deploy/templates/opa-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..501bc80a658f2cf38a12d5363a9db9eceace132a --- /dev/null +++ b/devops/core-plus/deploy/templates/opa-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: "{{ .Values.opa.conf.appName }}" + labels: + app: "{{ .Values.opa.conf.appName }}" + namespace: "{{ .Release.Namespace }}" +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 8181 + name: http + selector: + app: "{{ .Values.opa.conf.appName }}" diff --git a/devops/core-plus/deploy/templates/policy-configmap-bootstrap.yaml b/devops/core-plus/deploy/templates/policy-configmap-bootstrap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c91a6188928fab594c98cd508754090c8927ffe --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-configmap-bootstrap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.configmap }}-bootstrap" + namespace: "{{ .Release.Namespace }}" +data: + POLICY_BUCKET: "{{ .Values.data.bucketName }}" + PARTITION_BASE_URL: {{ .Values.data.partitionHost | quote }} + ONPREM_ENABLED: "true" diff --git a/devops/core-plus/deploy/templates/policy-configmap.yaml b/devops/core-plus/deploy/templates/policy-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85e9c593ea3edd8e26bdbbe0646887eac880f520 --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.configmap }}" + namespace: "{{ .Release.Namespace }}" +data: + LOG_LEVEL: {{ .Values.data.logLevel | default .Values.global.logLevel | quote }} + OPA_URL: {{ printf "http://%s" .Values.opa.conf.appName | quote }} + ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlementsHost }}" + ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlementsBasePath }}" + LEGAL_BASE_URL: "{{ .Values.data.legalHost }}" + POLICY_BUCKET: "{{ .Values.data.bucketName }}" + USE_BUNDLES: "{{ .Values.data.useBundles }}" + CLOUD_PROVIDER: "baremetal" + MINIO_ENDPOINT: "{{ .Values.data.minioHost }}" diff --git a/devops/core-plus/deploy/templates/policy-deployment-bootstrap.yaml b/devops/core-plus/deploy/templates/policy-deployment-bootstrap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea5e5a34a0ab3737e16050d3fd91508262193f35 --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-deployment-bootstrap.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: "{{ .Values.conf.appName }}-bootstrap" + type: bootstrap + name: "{{ .Values.conf.appName }}-bootstrap" + namespace: "{{ .Release.Namespace }}" +spec: + replicas: 1 + selector: + matchLabels: + app: "{{ .Values.conf.appName }}-bootstrap" + template: + metadata: + labels: + app: "{{ .Values.conf.appName }}-bootstrap" + annotations: + rollme: {{ randAlphaNum 5 | quote }} + sidecar.istio.io/proxyCPU: {{ .Values.istio.bootstrapProxyCPU | quote }} + sidecar.istio.io/proxyMemory: {{ .Values.istio.proxyMemory | quote }} + sidecar.istio.io/proxyCPULimit: {{ .Values.istio.bootstrapProxyCPULimit | quote }} + sidecar.istio.io/proxyMemoryLimit: {{ .Values.istio.proxyMemoryLimit | quote }} + spec: + containers: + - name: "{{ .Values.conf.appName }}-bootstrap" + image: "{{ .Values.data.bootstrapImage }}" + readinessProbe: + exec: + command: + - cat + - /tmp/bootstrap_ready + imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" + envFrom: + - configMapRef: + name: "{{ .Values.conf.configmap }}-bootstrap" + - secretRef: + name: "{{ .Values.conf.bootstrapSecretName }}" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + serviceAccountName: "{{ .Values.data.bootstrapServiceAccountName }}" diff --git a/devops/core-plus/deploy/templates/policy-deployment.yaml b/devops/core-plus/deploy/templates/policy-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb5e553f8ddfa75377c26084043e4b575d4a4fd1 --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-deployment.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: "{{ .Values.conf.appName }}" + type: core + source: python + name: "{{ .Values.conf.appName }}" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + app: "{{ .Values.conf.appName }}" + replicas: {{ .Values.conf.replicas }} + template: + metadata: + labels: + app: "{{ .Values.conf.appName }}" + annotations: + rollme: {{ randAlphaNum 5 | quote }} + sidecar.istio.io/proxyCPU: {{ .Values.istio.proxyCPU | quote }} + sidecar.istio.io/proxyMemory: {{ .Values.istio.proxyMemory | quote }} + sidecar.istio.io/proxyCPULimit: {{ .Values.istio.proxyCPULimit | quote }} + sidecar.istio.io/proxyMemoryLimit: {{ .Values.istio.proxyMemoryLimit | quote }} + spec: + containers: + - name: "{{ .Values.conf.appName }}" + image: "{{ .Values.data.image }}" + imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" + envFrom: + - configMapRef: + name: "{{ .Values.conf.configmap }}" + - secretRef: + name: "{{ .Values.conf.minioSecretName }}" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + ports: + - containerPort: 8080 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /api/policy/v1/health + port: 8080 + initialDelaySeconds: 190 + periodSeconds: 60 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /api/policy/v1/ready + port: 8080 + periodSeconds: 60 + resources: + requests: + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" + {{- if .Values.global.limitsEnabled }} + limits: + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" + {{- end }} + serviceAccountName: "{{ .Values.data.serviceAccountName }}" + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 diff --git a/devops/core-plus/deploy/templates/policy-service-account.yaml b/devops/core-plus/deploy/templates/policy-service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46a64baf68062e9d30bce2f770892a54a651562c --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Values.data.serviceAccountName }}" + namespace: "{{ .Release.Namespace }}" diff --git a/devops/core-plus/deploy/templates/policy-service.yaml b/devops/core-plus/deploy/templates/policy-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a56e209a91c1a3b3f0c258ed31c59fc8df1dbfef --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Values.conf.appName }}" + namespace: "{{ .Release.Namespace }}" + labels: + app: "{{ .Values.conf.appName }}" + service: "{{ .Values.conf.appName }}" +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + name: http + selector: + app: "{{ .Values.conf.appName }}" diff --git a/devops/core-plus/deploy/templates/policy-virtual-service.yaml b/devops/core-plus/deploy/templates/policy-virtual-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a14cf423aa9e7188eac4cf1d209ca61555b6d5e1 --- /dev/null +++ b/devops/core-plus/deploy/templates/policy-virtual-service.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: "{{ .Values.conf.appName }}" + namespace: "{{ .Release.Namespace }}" +spec: + hosts: + {{- if .Values.global.domain }} + - {{ printf "osdu.%s" .Values.global.domain | quote }} + {{- else }} + - "*" + {{- end }} + gateways: + - service-gateway + http: + - match: + - uri: + prefix: "/api/policy/v1" + route: + - destination: + port: + number: 80 + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" diff --git a/devops/core-plus/deploy/values.yaml b/devops/core-plus/deploy/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2baa5b527866572a0cb4c795c4332ab18c0d796 --- /dev/null +++ b/devops/core-plus/deploy/values.yaml @@ -0,0 +1,53 @@ +# Common values for all deployments +global: + domain: '' + limitsEnabled: true + dataPartitionId: "" + logLevel: "ERROR" + +data: + # Deployment resources + requestsCpu: 5m + requestsMemory: 1.2G + limitsCpu: '1' + limitsMemory: 1.5G + serviceAccountName: policy + imagePullPolicy: IfNotPresent + image: '' + bootstrapImage: '' + bootstrapServiceAccountName: '' + # ConfigMap resources + logLevel: "" + entitlementsHost: http://entitlements + entitlementsBasePath: /api/entitlements/v2/groups + legalHost: http://legal + partitionHost: http://partition + bucketName: "refi-opa-policies" + useBundles: 'yes' + minioHost: http://minio:9000 +conf: + appName: policy + configmap: policy-config + minioSecretName: policy-minio-secret + bootstrapSecretName: minio-bootstrap-secret + replicas: 1 +opa: + data: + requestsMemory: 200Mi + image: docker.io/openpolicyagent/opa:0.70.0 + serviceAccountName: opa + conf: + envConfig: opa-env-config + appName: opa + replicas: 1 + init: + name: init-config + image: '' + +istio: + proxyCPU: 10m + proxyCPULimit: 500m + proxyMemory: 50Mi + proxyMemoryLimit: 512Mi + bootstrapProxyCPU: 5m + bootstrapProxyCPULimit: 100m diff --git a/devops/core-plus/init-container-opa/Dockerfile b/devops/core-plus/init-container-opa/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6eecf90461104a9a90a1d5c9caf7b1672f184692 --- /dev/null +++ b/devops/core-plus/init-container-opa/Dockerfile @@ -0,0 +1,18 @@ +FROM alpine:latest + +WORKDIR /opt + +COPY ./devops/core-plus/init-container-opa/ /opt/ + +RUN chmod 775 init_opa.sh validate-env.sh + +RUN apk add jq bash curl + +RUN addgroup -g 10001 -S nonroot \ + && adduser -h /opt -G nonroot -S -u 10001 nonroot + +RUN chown -R 10001:10001 /opt + +USER 10001:10001 + +CMD ["/bin/bash", "-c", "/opt/init_opa.sh"] diff --git a/devops/core-plus/init-container-opa/init_opa.sh b/devops/core-plus/init-container-opa/init_opa.sh new file mode 100644 index 0000000000000000000000000000000000000000..fa9cca4141118728363f50fc0224ffcedd1ae5d8 --- /dev/null +++ b/devops/core-plus/init-container-opa/init_opa.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +source ./validate-env.sh "PARTITION_BASE_URL" +source ./validate-env.sh "POLICY_BUCKET" +source ./validate-env.sh "ONPREM_ENABLED" + +# Get all partitions +status_code=$(curl -X GET \ + --url "${PARTITION_BASE_URL}/api/partition/v1/partitions" --write-out "%{http_code}" --silent --output "/dev/null" \ + -H "Content-Type: application/json") + +if [[ "${status_code}" == 200 ]]; then + PARTITIONS_LIST=$(curl --location "${PARTITION_BASE_URL}/api/partition/v1/partitions" | jq -r '[.[] | select(. != "system")] | join(",")') + IFS=',' read -ra PARTITIONS <<< "${PARTITIONS_LIST}" + echo $PARTITIONS + # Check for partition bootstrap + if [ -z "$PARTITIONS" ] + then + echo "Partition bootstrap is not finished" + sleep 15 + exit 1 + fi + else + echo "Exiting with status code: ${status_code}" + sleep 15 + exit 1 + fi + +# Function for create config.yaml for OPA +create_config_gc() { +echo "Creating config file for OPA..." +cat << EOF > /config/config.yaml +services: + gcs: + url: "https://storage.googleapis.com/storage/v1/b/$POLICY_BUCKET/o" + credentials: + gcp_metadata: + scopes: + - "https://www.googleapis.com/auth/devstorage.read_only" +bundles: + osdu/instance: + service: gcs + # NOTE ?alt=media is required + resource: 'bundle.tar.gz?alt=media' +EOF + +for PARTITION in "${PARTITIONS[@]}"; do +cat << EOF >> /config/config.yaml + osdu/partition/$PARTITION: + service: gcs + resource: 'bundle-$PARTITION.tar.gz?alt=media' + polling: + min_delay_seconds: 6 + max_delay_seconds: 12 +EOF +done +echo "Config for OPA is ready" +} + +create_config_onprem() { +echo "Creating config file for OPA..." +cat << EOF > /config/config.yaml +services: + s3: + url: http://minio:9000/$POLICY_BUCKET + credentials: + s3_signing: + environment_credentials: {} +bundles: + osdu/instance: + service: s3 + resource: bundle.tar.gz +EOF + +for PARTITION in "${PARTITIONS[@]}"; do +cat << EOF >> /config/config.yaml + osdu/partition/$PARTITION: + service: s3 + resource: 'bundle-$PARTITION.tar.gz' +EOF +done +echo "Config for OPA is ready" +} + +if [ "${ONPREM_ENABLED}" == "true" ]; then + create_config_onprem +else + create_config_gc +fi diff --git a/devops/core-plus/init-container-opa/validate-env.sh b/devops/core-plus/init-container-opa/validate-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..9de17ff4a83b4e297bba6aec875088dd313c416b --- /dev/null +++ b/devops/core-plus/init-container-opa/validate-env.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +{ set +x ;} 2> /dev/null # disable output to prevent secret logging +set -e + +ENV_VAR_NAME=$1 + +if [ "${!ENV_VAR_NAME}" = "" ] +then + echo "Missing environment variable '$ENV_VAR_NAME'. Please provide all variables and try again" + { set -x ;} 2> /dev/null # enable output back + exit 1 +fi + +{ set -x ;} 2> /dev/null # enable output back diff --git a/devops/core-plus/pipeline/override-stages.yml b/devops/core-plus/pipeline/override-stages.yml new file mode 100644 index 0000000000000000000000000000000000000000..c3f3115ec016ec20e8e9f88a7e001a2224046d42 --- /dev/null +++ b/devops/core-plus/pipeline/override-stages.yml @@ -0,0 +1,89 @@ +variables: + CORE_BUILD_PATH: "build/core-plus/Dockerfile" + CORE_BUILD_BOOTSTRAP_PATH: "devops/core-plus/bootstrap-osdu-module/Dockerfile" + CORE_SERVICE: policy + CORE_ENABLE_BOOTSTRAP: "true" + CORE_INT_TEST_TYPE: python + CORE_BAREMETAL_PYTHON_INT_TEST_SUBDIR: "app/tests/baremetal" + CORE_INIT_IMAGE_NAME: core-plus-opa-init-container + CORE_SUPPORTED_PLATFORM: "linux/amd64" + +# Workaround since tests by default extends .maven. Need it for tests when disabling other CSPs +.maven: + variables: + +.core_set_image_name: + script: + - > + if echo $CI_COMMIT_REF_NAME | grep -Eq "^release\/[0-9]{1,2}.[0-9]{1,2}$"; + then + export CORE_IMAGE_NAME=$CORE_IMAGE_NAME-release; + export CORE_IMAGE_BOOTSTRAP_NAME=$CORE_IMAGE_BOOTSTRAP_NAME-release; + export CORE_INIT_IMAGE_NAME=$CORE_INIT_IMAGE_NAME-release; + fi + - > + if [[ "$CI_COMMIT_REF_NAME" == "$CI_DEFAULT_BRANCH" ]]; + then + export CORE_IMAGE_NAME=$CORE_IMAGE_NAME-master; + export CORE_IMAGE_BOOTSTRAP_NAME=$CORE_IMAGE_BOOTSTRAP_NAME-master; + export CORE_INIT_IMAGE_NAME=$CORE_INIT_IMAGE_NAME-master; + fi + - > + if [[ "$CI_COMMIT_TAG" != "" ]]; + then + CORE_IMAGE_TAG="$CI_COMMIT_TAG"; + CORE_EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_NAME:$CI_COMMIT_TAG"; + CORE_EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_TAG"; + CORE_EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$CORE_INIT_IMAGE_NAME:$CI_COMMIT_TAG"; + elif [[ "$CI_COMMIT_REF_NAME" == "$CI_DEFAULT_BRANCH" ]]; + then + CORE_IMAGE_TAG="$CI_COMMIT_SHORT_SHA"; + HELM_TAG="latest" + CORE_EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$CORE_IMAGE_NAME:latest"; + CORE_EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$CORE_IMAGE_BOOTSTRAP_NAME:latest"; + CORE_EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$CORE_INIT_IMAGE_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$CORE_INIT_IMAGE_NAME:latest"; + else + CORE_IMAGE_TAG="$CI_COMMIT_SHORT_SHA"; + CORE_HELM_TAG="core$CI_COMMIT_SHORT_SHA" + CORE_EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_NAME:$CI_COMMIT_SHORT_SHA"; + CORE_EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$CORE_IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_SHORT_SHA"; + CORE_EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$CORE_INIT_IMAGE_NAME:$CI_COMMIT_SHORT_SHA"; + fi + +.core_substitute_image_in_helm: + script: + - wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq + - IMAGE="$CI_REGISTRY_IMAGE/$CORE_IMAGE_NAME:$CORE_IMAGE_TAG" yq -i '.data.image = env(IMAGE)' $CORE_HELM_DEPLOYMENT_DIR/values.yaml + - yq -e '.data | has("bootstrapImage")' $CORE_HELM_DEPLOYMENT_DIR/values.yaml &>/dev/null && BOOTSTRAP_IMAGE="$CI_REGISTRY_IMAGE/$CORE_IMAGE_BOOTSTRAP_NAME:$CORE_IMAGE_TAG" yq -i '.data.bootstrapImage = env(BOOTSTRAP_IMAGE)' $CORE_HELM_DEPLOYMENT_DIR/values.yaml + - CORE_INIT_IMAGE="$CI_REGISTRY_IMAGE/$CORE_INIT_IMAGE_NAME:$CORE_IMAGE_TAG" yq -i '.opa.init.image = env(CORE_INIT_IMAGE)' $CORE_HELM_DEPLOYMENT_DIR/values.yaml + - cat $CORE_HELM_DEPLOYMENT_DIR/values.yaml | grep -i image + +core-containerize-gitlab: + needs: + - "compile-and-unit-test" + +core-containerize-init-gitlab: + needs: + - "compile-and-unit-test" + extends: core-containerize-gitlab + variables: + BUILD_PATH: "devops/core-plus/init-container-opa/Dockerfile" + script: + - docker run --privileged --rm tonistiigi/binfmt --install all + - docker buildx create --driver docker-container --use + - docker buildx inspect --bootstrap + - docker buildx build --provenance=false --platform "$CORE_SUPPORTED_PLATFORM" $CORE_EXTRA_INIT_TAG -f "$BUILD_PATH" $CORE_BUILD_ARGS --push . + +core-test-python: + image: python:3.11 + variables: + POLICY_BUCKET: "refi-opa-policies" + CLOUD_PROVIDER: "baremetal" + OPA_URL: "policy" + ENTITLEMENTS_BASE_URL: $HOST + LEGAL_BASE_URL: $HOST + PARTITION_BASE_URL: $HOST + MINIO_ENDPOINT: $TEST_MINIO_URL + MINIO_ACCESS_KEY: $TEST_MINIO_ACCESS_KEY + MINIO_SECRET_KEY: $TEST_MINIO_SECRET_KEY + DATA_PARTITION: "osdu"