Merge branch 'aws-helm' into 'master'

Adding AWS Helm Charts

See merge request !257

devops/aws/chart/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Build files
+*.html
+build-info.json
\ No newline at end of file

devops/aws/chart/Chart.yaml

+apiVersion: v2
+name: os-partition
+version: 0.1.0
+kubeVersion: "v1.21.x-x-x"
+description: Partitions Helm chart for Kubernetes
+type: application
+appVersion: __VERSION__
+dependencies:
+  - name: osdu-aws-lib
+    version: 0.1.0
+    repository:  __HELM_REPO__/osdu-aws-lib/
+deprecated: false

devops/aws/chart/README.md

+# Helm Chart
+
+## Introduction
+The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager.
+
+## Prerequisites
+The below software must be installed before continuing:
+* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/)
+* [Helm ^3.7.1](https://helm.sh/docs/intro/install/)
+* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3)
+
+Additionally, an OSDU on AWS environment must be deployed.
+
+## Installation/Updating
+To install or update the service application by executing the following command in the CHART folder:
+
+```bash
+helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE]
+```
+
+To observe the Kubernetes resources before deploying them using the command:
+```bash
+helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug
+```
+
+To observe the history of the current release, use the following command:
+```bash
+helm history [RELEASE_NAME] -n [NAMESPACE]
+```
+
+To revert to a previous release, use the following command:
+```bash
+helm rollback [RELEASE] [REVISION] -n [NAMESPACE]
+```
+
+### Customizing the Deployment
+It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration:
+
+| Name | Example Value | Description | Type | Required |
+| ---  | ------------- | ----------- | ---- | -------- |
+| `global.accountID` | `000123456789` | The AWS account ID. | int | yes |
+| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes |
+| `global.resourcePrefix` | `osdu` | The resource prefix of the OSDU deployment. | str | yes |
+| `global.allowOrigins` | `{http://localhost,https://www.osdu.aws}` | A list of domains that are permitted by CORS policy. An empty list permits all origins. | array[str] | no |
+| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no |
+| `replicaCount` | `1` | The number of pod replicas to be deployed | int | no |
+| `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no |
+| `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no |
+| `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no |
+
+## Uninstalling the Chart
+To uninstall the helm release:
+
+```bash
+helm uninstall [RELEASE] -n [NAMESPACE] --keep-history
+```
\ No newline at end of file

devops/aws/chart/templates/NOTES.txt

+{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace
+{{- if (.Values.service).apiPath }}
+Application URL available at path {{ .Values.service.apiPath }}
+{{- end }}
\ No newline at end of file

devops/aws/chart/templates/authorizationpolicy.yaml

+{{- template "common.authzpolicy" (list . "os-partition.authzpolicy") -}}
+{{- define "os-partition.authzpolicy" -}}
+## Define overrides for the service authorization policy resource here
+{{- end -}}
+

devops/aws/chart/templates/deployment.yaml

+{{- template "common.deployment" (list . "os-partition.deployment") -}}
+{{- define "os-partition.deployment" -}}
+## Define overrides for the service deployment resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/destinationrule.yaml

+{{- template "common.destinationrule" (list . "os-partition.destinationrule") -}}
+{{- define "os-partition.destinationrule" -}}
+## Define overrides for the service destination rule resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/hpa.yaml

+{{- template "common.hpa" (list . "os-partition.hpa") -}}
+{{- define "os-partition.hpa" -}}
+## Define overrides for the service HPA resource here
+{{- end -}}

devops/aws/chart/templates/service.yaml

+{{- template "common.service" (list . "os-partition.service") -}}
+{{- define "os-partition.service" -}}
+## Define overrides for the service resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/serviceaccount.yaml

+{{- template "common.serviceaccount" (list . "os-partition.serviceaccount") -}}
+{{- define "os-partition.serviceaccount" -}}
+## Define overrides for the service's service account resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/serviceparameters.yaml

+{{- template "common.serviceparameters" (list . "os-partition.serviceparameters") -}}
+{{- define "os-partition.serviceparameters" -}}
+## Define overrides for the service parameters here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/tests/test-connection.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "common.fullname" . }}-test-connection"
+  labels:
+    {{- include "common.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "common.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

devops/aws/chart/templates/virtualservice.yaml

+{{- template "common.virtualservice" (list . "os-partition.virtualservice") -}}
+{{- define "os-partition.virtualservice" -}}
+## Define overrides for the service's virtual service resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/values.schema.json

+{
+    "$schema": "osdu-aws-services-helm",
+    "type": "object",
+    "default": {},
+    "required": [
+        "image",
+        "imagePullPolicy",
+        "service",
+        "podAnnotations",
+        "replicaCount",
+        "serviceAccountRole",
+        "securityContext"
+    ],
+    "properties": {
+        "image": {
+            "type": "string",
+            "title": "The service container image URL"
+        },
+        "imagePullPolicy": {
+            "type": "string",
+            "pattern": "^(Always|Never|IfNotPresent)$",
+            "default": "Always"
+        },
+        "service": {
+            "type": "object",
+            "required": [
+                "type",
+                "port",
+                "apiPath"
+            ],
+            "properties": {
+                "type": {
+                    "type": "string",
+                    "default": "ClusterIP"
+                },
+                "port": {
+                    "type": "integer",
+                    "default": 8080,
+                    "title": "The port the service is hosted on",
+                    "examples": [
+                        8080,
+                        443,
+                        80
+                    ]
+                },
+                "apiPath": {
+                    "type": "string",
+                    "title": "The main API path",
+                    "examples": [
+                        "/api/service/v1/"
+                    ]
+                },
+                "readinessProbeHttpPath": {
+                    "type": "string",
+                    "examples": [
+                        "/api/service/v1/actuator/health"
+                    ]
+                },
+                "livenessProbeHttpPath": {
+                    "type": "string",
+                    "examples": [
+                        "/api/service/v1/actuator/health"
+                    ]
+                },
+                "startupProbeHttpPath": {
+                    "type": "string",
+                    "examples": [
+                        "/api/service/v1/actuator/health"
+                    ]
+                }
+            },
+            "examples": [{
+                "type": "ClusterIP",
+                "port": 8080,
+                "apiPath": "/api/service/v1/",
+                "readinessProbeHttpPath": "/api/service/v1/actuator/health",
+                "livenessProbeHttpPath": "/api/service/v1/actuator/health",
+                "startupProbeHttpPath": "/api/service/v1/actuator/health"
+            }]
+        },
+        "serviceParameters": {
+            "type": "array",
+            "title": "The secret parameters consumed by the service",
+            "items": {
+                "type": "object",
+                "required": [
+                    "objectName",
+                    "objectType",
+                    "objectAlias"
+                ],
+                "properties": {
+                    "objectName": {
+                        "type": "string",
+                        "title": "The SSM or secrets parameter name",
+                        "examples": [
+                            "/osdu/resource-prefix/redis/redisauthtoken",
+                            "/osdu/resource-prefix/redis-core/end-point"
+                        ]
+                    },
+                    "objectType": {
+                        "type": "string",
+                        "title": "The type of parameter",
+                        "pattern": "^(secretsmanager|ssmparameter|)$"
+                    },
+                    "objectAlias": {
+                        "type": "string",
+                        "title": "The name of the environment variable consumed by the service",
+                        "examples": [
+                            "CACHE_CLUSTER_KEY",
+                            "CACHE_CLUSTER_ENDPOINT"
+                        ]
+                    }
+                },
+                "examples": [{
+                    "objectName": "/osdu/resource-prefix/redis/redisauthtoken",
+                    "objectType": "secretsmanager",
+                    "objectAlias": "CACHE_CLUSTER_KEY"
+                },
+                {
+                    "objectName": "/osdu/resource-prefix/redis-core/end-point",
+                    "objectType": "ssmparameter",
+                    "objectAlias": "CACHE_CLUSTER_ENDPOINT"
+                }]
+            }
+        },
+        "environmentVariables": {
+            "type": "array",
+            "title": "The environment variables consumed by the service",
+            "items": {
+                "type": "object",
+                "title": "An environment variable definition",
+                "required": [
+                    "name",
+                    "value"
+                ],
+                "properties": {
+                    "name": {
+                        "type": "string",
+                        "title": "The name of the environment variable consumed by the service",
+                        "examples": [
+                            "APPLICATION_PORT",
+                            "AWS_REGION"
+                        ]
+                    },
+                    "value": {
+                        "type": "string",
+                        "title": "The value of the environment variable"
+                    }
+                },
+                "examples": [{
+                    "name": "APPLICATION_PORT",
+                    "value": "8080"
+                },
+                {
+                    "name": "AWS_REGION",
+                    "value": "us-east-1"
+                }]
+            }
+        },
+        "podAnnotations": {
+            "type": "object",
+            "default": {}
+        },
+        "replicaCount": {
+            "type": "integer",
+            "default": 1,
+            "title": "The number of pod replicas"
+        },
+        "maxConnections": {
+            "type": "integer",
+            "default": 200,
+            "title": "The max number of service connections"
+        },
+        "resources": {
+            "type": "object"
+        },
+        "autoscaling": {
+            "type": "object"
+        },
+        "serviceAccountRole": {
+            "type": "string",
+            "default": "",
+            "title": "The service account IAM role"
+        },
+        "cors": {
+            "type": "object",
+            "default": {},
+            "title": "The cors Schema",
+            "required": [
+                "maxAge",
+                "allowCredentials",
+                "allowMethods",
+                "allowHeaders"
+            ],
+            "properties": {
+                "maxAge": {
+                    "type": "string",
+                    "default": "60m",
+                    "title": "The max age of the request"
+                },
+                "allowCredentials": {
+                    "type": "boolean",
+                    "default": true,
+                    "title": "The allowCredentials Schema"
+                },
+                "allowMethods": {
+                    "type": "array",
+                    "title": "A list of supported API request methods",
+                    "items": {
+                        "type": "string",
+                        "title": "Supported API request methods",
+                        "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$"
+                    },
+                    "examples": [
+                        ["POST",
+                            "GET",
+                            "PATCH",
+                            "DELETE"
+                        ]
+                    ]
+                },
+                "allowHeaders": {
+                    "type": "array",
+                    "title": "A list of supported headers",
+                    "items": {
+                        "type": "string"
+                    },
+                    "examples": [
+                        ["Authorization",
+                            "Data-Partition-Id",
+                            "Correlation-Id",
+                            "Content-Type"
+                        ]
+                    ]
+                }
+            },
+            "examples": [{
+                "maxAge": "60m",
+                "allowCredentials": true,
+                "allowMethods": [
+                    "POST",
+                    "GET",
+                    "PATCH",
+                    "DELETE"
+                ],
+                "allowHeaders": [
+                    "Authorization",
+                    "Data-Partition-Id",
+                    "Correlation-Id",
+                    "Content-Type"
+                ]
+            }]
+        },
+        "securityContext": {
+            "type": "object",
+            "default": {}
+        },
+        "allowedPrincipals": {
+            "type": "array",
+            "title": "A list of allowed service principals",
+            "items": {
+                "type": "string",
+                "title": "Allowed principal",
+                "examples": [
+                    "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account",
+                    "cluster.local/ns/osdu-services/sa/compliance-queue"
+                ]
+            }
+        }
+    }
+}
\ No newline at end of file

devops/aws/chart/values.yaml

+# Service Config
+image: __CONTAINER__
+imagePullPolicy: IfNotPresent
+service:
+  type: ClusterIP
+  port: 8080
+  apiPath: /api/partition/v1/
+  readinessProbeHttpPath: /api/partition/v1/actuator/health
+  livenessProbeHttpPath: /api/partition/v1/actuator/health
+serviceParameters:
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis/redisauthtoken"
+    objectType: "secretsmanager"
+    objectAlias: "CACHE_CLUSTER_KEY"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis-core/end-point"
+    objectType: "ssmparameter"
+    objectAlias: "CACHE_CLUSTER_ENDPOINT"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis-core/end-point-port"
+    objectType: "ssmparameter"
+    objectAlias: "CACHE_CLUSTER_PORT"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/primary-region"
+    objectType: "ssmparameter"
+    objectAlias: primary-region
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/mongodb/endpoint"
+    objectType: "ssmparameter"
+    objectAlias: "mongodb_host"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/mongodb/port"
+    objectType: "ssmparameter"
+    objectAlias: "mongodb_port"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/mongodb/credentials"
+    objectType: "secretsmanager"
+    objectAlias: "mongodb_credentials"
+  - objectName: "/osdu/{{ .Values.global.resourcePrefix }}/partition/DataKmsKey/arn"
+    objectType: "ssmparameter"
+    objectAlias: "KEY_ARN"
+environmentVariables:
+  - name: APPLICATION_PORT
+    value: "{{ .Values.service.port }}"
+  - name: AWS_REGION
+    value: "{{ .Values.global.region }}"
+  - name: ENVIRONMENT
+    value: "{{ .Values.global.resourcePrefix }}"
+  - name: JAVA_OPTS
+    value: "-Xms538M -Xmx900M"
+  - name: LOG_LEVEL
+    value: "{{ default `INFO` .Values.global.logLevel }}"
+  - name: SSM_ENABLED
+    value: "True"
+  - name: SSL_ENABLED
+    value: "false"
+  - name: ENTITLEMENTS_BASE_URL
+    value: "http://os-entitlements:8080"
+  - name: PARAMETER_MOUNT_PATH
+    value: "/mnt/params"
+  - name: MONGODB_AUTH_DATABASE
+    value: "admin"
+  - name: MONGODB_RETRY_WRITES
+    value: "true"
+  - name: MONGODB_WRITE_MODE
+    value: "majority"
+  - name: MONGODB_USE_SRV_ENDPOINT
+    value: "true"
+  - name: MONGODB_ENABLE_TLS
+    value: "false"
+podAnnotations: {}
+# Resource Config
+replicaCount: 1
+resources:
+  limits:
+    memory: 900M
+  requests:
+    cpu: 500m
+    memory: 900M
+autoscaling:
+  enabled: true
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+# Security Config
+serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/osdu-{{ .Values.global.resourcePrefix }}-{{ include "common.name" . }}
+cors:
+  maxAge: "60m"
+  allowCredentials: true
+  allowMethods:
+    - POST
+    - GET
+    - PATCH
+    - DELETE
+  allowHeaders:
+    - Authorization
+    - Data-Partition-Id
+    - Correlation-Id
+    - Content-Type
+securityContext: {}
+# capabilities:
+#   drop:
+#   - ALL
+# readOnlyRootFilesystem: true
+# runAsNonRoot: true
+# runAsUser: 1000
+allowedPrincipals:
+  - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/compliance-queue
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/compliance-queue-trigger
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/indexer-queue
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-crs-catalog
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-crs-conversion
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-dataset
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-entitlements
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-file
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-file-dms
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-indexer
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-legal
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-notification
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-partition
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-policy
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-register
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-schema
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-search
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-storage
+  - cluster.local/ns/{{ .Release.Namespace }}/sa/os-unit
+  - cluster.local/ns/os-timeseries-dms/sa/os-timeseries-dms
+  - cluster.local/ns/aws-binary-dms/sa/binary-dms
+  - cluster.local/ns/osdu-airflow/sa/airflow-dag-upload
+  - cluster.local/ns/osdu-airflow/sa/airflow-worker
+  - cluster.local/ns/osdu-ingest/sa/os-data-workflow
+  - cluster.local/ns/osdu-ingest/sa/os-ingestion-workflow
+  - cluster.local/ns/osdu-seismic-ddms/sa/os-seismic-store
+  - cluster.local/ns/osdu-well-delivery/sa/os-welldelivery
+  - cluster.local/ns/osdu-wellbore-ddms/sa/os-wellbore-ddms

provider/partition-aws/build-aws/buildspec.yaml

@@ -68,6 +68,15 @@ phases:
       - echo "Building integration testing assemblies and gathering artifacts..."
       - ./testing/partition-test-aws/build-aws/prepare-dist.sh
 
+      # Push Helm charts to repo
+      # - cd devops/charts
+      # - for dir in */ ; do chart=$(echo ${dir} | sed 's:/*$::'); helm s3 init ${HELM_REPO}/${BRANCH_NAME}/${chart}; helm repo add --force-update ${chart} ${HELM_REPO}/${BRANCH_NAME}/${chart}; helm package ./${chart}; helm s3 push --force $(find . -name ${chart}'-*.tgz' -print | head -n 1) ${chart}; done
+      # - cd ../..
+
+      - echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--"
+      - rsync -r devops/aws/* "${OUTPUT_DIR:-dist}"
+
+
       - echo "Logging into Docker Hub..."
       - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}