Merge branch 'aws-infra-merge' into 'master'

M18 service and CICD updates

See merge request !385

.gitlab-ci.yml

@@ -5,7 +5,7 @@ variables:
   AWS_SERVICE: notification
   AWS_SERVICE_GATEWAY: osdu-gateway
   AWS_ENVIRONMENT: dev
-  AWS_DEPLOY_TARGET: HELM
+  AWS_DEPLOY_TARGET: TF
   AWS_EKS_DEPLOYMENT_NAME: os-notification
 
   AZURE_SERVICE: notification

devops/aws/chart/README.md

-# Helm Chart
+# OSDU on AWS Service Helm Chart
 
 ## Introduction
 The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager.
 
 ## Prerequisites
 The below software must be installed before continuing:
-* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
-* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/)
-* [Helm ^3.7.1](https://helm.sh/docs/intro/install/)
-* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3)
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+* [kubectl](https://kubernetes.io/docs/tasks/tools/)
+* [Helm](https://helm.sh/docs/intro/install/)
+* [Helm S3 Plugin](https://github.com/hypnoglow/helm-s3)
 
 Additionally, an OSDU on AWS environment must be deployed.
 
@@ -34,21 +34,55 @@ To revert to a previous release, use the following command:
 helm rollback [RELEASE] [REVISION] -n [NAMESPACE]
 ```
 
-### Customizing the Deployment
+Refer to the [Helm CLI guide](https://helm.sh/docs/helm/helm/) for additional commands.
+
+## Customizing the Deployment
 It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration:
 
+### Globals
+Global Helm values apply to all services within the parent chart deployment. Global values will not override service defaults or locally set values.
 | Name | Example Value | Description | Type | Required |
 | ---  | ------------- | ----------- | ---- | -------- |
-| `global.accountID` | `000123456789` | The AWS account ID. | int | yes |
-| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes |
-| `global.resourcePrefix` | `osdu` | The resource prefix of the OSDU deployment. | str | yes |
 | `global.allowOrigins` | `{http://localhost,https://www.osdu.aws}` | A list of domains that are permitted by CORS policy. An empty list permits all origins. | array[str] | no |
-| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `global.metricsServerAddress` | `http://prometheus-service.monitoring:8080` | The URL of the accessible metrics server for evaluating autoscaling decisions. | str | no |
+| `global.podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `global.podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
+| `global.securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
+
+### Local
+Local Helm values apply to specific services. Local Helm values will override global values and default presets.
+| Name | Example Value | Description | Type | Required |
+| ---  | ------------- | ----------- | ---- | -------- |
+| `image` | `registry.repo.osdu.aws/service:0.21.0` | The custom image of the service deployment. | str | no |
 | `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no |
-| `replicaCount` | `1` | The number of pod replicas to be deployed | int | no |
+| `resources.limits.cpu` | `500M` | [CPU resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.limits.memory` | `900M` | [Memory resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.requests.cpu` | `500M` | [MemoCPUry resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.requests.memory` | `900M` | [Memory resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `replicaCount` | `1` | The number of pod replicas to be initially deployed | int | no |
 | `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no |
 | `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no |
 | `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no |
+| `autoscaling.targetMemoryUtilizationPercentage` | `80` | Memory utilization target | int | no |
+| `autoscaling.ServiceRequestCountThreshold` | `25` | The number of requests per second threshold averaged over a minute to trigger a scaling event. | int | no |
+| `autoscaling.ServiceRequestDurationAverage` | `300` | The response time measured in miliseconds averaged over 3 minutes to trigger a scaling event. | int | no |
+| `autoscaling.coolDownPeriod` | `120` | The period to wait after the last trigger reported active before scaling the resource back to 0. Managed by Keda. | int | no |
+| `autoscaling.pollingInterval` | `1` | This is the interval to check each trigger on. | int | no |
+| `livenessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.initialDelaySeconds` | `30` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `maxPendingRequests` | `10000` | Maximum number of requests that will be queued while waiting for a ready connection pool connection. Used for circuit breaking. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `maxRequestsPerConnection` | `100` | Maximum number of active requests to a destination. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `maxConnections` | `0` | Maximum number of HTTP1 /TCP connections to a destination host. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
+| `securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
 
 ## Uninstalling the Chart
 To uninstall the helm release:

devops/aws/chart/templates/hpa.yaml

-{{- template "common.hpa" (list . "os-notification.hpa") -}}
-{{- define "os-notification.hpa" -}}
-## Define overrides for the service HPA resource here
-{{- end -}}

devops/aws/chart/templates/scaledobject.yaml

+{{- template "common.scaledobject" (list . "os-notification.scaledobject") -}}
+{{- define "os-notification.scaledobject" -}}
+## Define overrides for the service's scaled object resource here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/templates/serviceparameters.yaml

+{{- template "common.serviceparameters" (list . "os-notification.serviceparameters") -}}
+{{- define "os-notification.serviceparameters" -}}
+## Define overrides for the service parameters here
+{{- end -}}
\ No newline at end of file

devops/aws/chart/values.schema.json

 {
-    "$schema": "osdu-aws-services-helm",
+    "$schema": "osdu-aws-services-public",
     "type": "object",
     "default": {},
     "required": [
         "image",
-        "imagePullPolicy",
         "service",
-        "replicaCount",
         "serviceAccountRole"
     ],
     "properties": {
@@ -22,8 +20,6 @@
         "service": {
             "type": "object",
             "required": [
-                "type",
-                "port",
                 "apiPath"
             ],
             "properties": {
@@ -260,10 +256,9 @@
                 "type": "string",
                 "title": "Allowed principal",
                 "examples": [
-                    "cluster.local/ns/istio-system/sa/istio-ingressgateway",
-                    "cluster.local/ns/osdu-services/sa/compliance-queue"
+                    "cluster.local/ns/istio-system/sa/istio-ingressgateway"
                 ]
             }
         }
     }
-}
\ No newline at end of file
+}

devops/aws/chart/values.yaml

-# Service Config
-image: __CONTAINER__
-imagePullPolicy: Always
-service:
-  type: ClusterIP
-  port: 8080
-  apiPath: /api/notification/v1/
-  readinessProbeHttpPath: /api/notification/v1/actuator/health
-  livenessProbeHttpPath: /api/notification/v1/actuator/health
-
-environmentVariables:
-  - name: APPLICATION_PORT
-    value: "{{ .Values.service.port }}"
-  - name: AWS_REGION
-    value: "{{ .Values.global.region }}"
-  - name: RESOURCE_PREFIX
-    value: "{{ .Values.global.resourcePrefix }}"
-  - name: JAVA_OPTS
-    value: "-Xms538M -Xmx900M"
-  - name: LOG_LEVEL
-    value: "{{ default `INFO` .Values.global.logLevel }}"
-  - name: SSM_ENABLED
-    value: "True"
-  - name: SSL_ENABLED
-    value: "false"
-  - name: ENTITLEMENTS_BASE_URL
-    value: "http://os-entitlements:8080"
-  - name: REGISTER_BASE_URL
-    value: http://os-register:8080
-  - name: TMP_VOLUME_PATH
-    value: "/tmp"
-  - name: PARTITION_BASE_URL
-    value: http://os-partition:8080
-
-# Resource Config
-replicaCount: 1
-resources:
-  limits:
-    memory: 900M
-  requests:
-    cpu: 500m
-    memory: 900M
-autoscaling:
-  enabled: true
-  minReplicas: 1
-  maxReplicas: 100
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
-
-# Security Config
-serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/osdu-{{ .Values.global.resourcePrefix }}-{{ .Values.global.region }}-{{ include "common.name" . }}
-cors:
-  maxAge: "60m"
-  allowCredentials: true
-  allowMethods: 
-    - POST
-    - GET
-    - PATCH
-    - PUT
-    - DELETE
-  allowHeaders:
-    - Authorization
-    - Data-Partition-Id
-    - Correlation-Id
-    - Content-Type
-securityContext: 
-  runAsUser: 10001
-  runAsNonRoot: true
-  readOnlyRootFilesystem: true
-  allowPrivilegeEscalation: false
-  capabilities:
-    drop:
-    - ALL
-podSecurityContext: 
-  fsGroup: 1337
-  seccompProfile:
-    type: RuntimeDefault
-
-allowedPrincipals:
-  - cluster.local/ns/istio-system/sa/istio-ingressgateway
-  - cluster.local/ns/aws-binary-dms/sa/binary-dms
-  - cluster.local/ns/osdu-airflow/sa/airflow-dag-upload
-  - cluster.local/ns/osdu-ingest/sa/os-data-workflow
-  - cluster.local/ns/osdu-ingest/sa/os-ingestion-workflow
-  - cluster.local/ns/osdu-seismic-ddms/sa/os-seismic-store
-  - cluster.local/ns/osdu-well-delivery/sa/os-welldelivery
-  - cluster.local/ns/osdu-wellbore-ddms/sa/os-wellbore-ddms
+image: __CONTAINER__
\ No newline at end of file

provider/notification-aws/build-aws/buildspec.yaml

@@ -81,7 +81,6 @@ phases:
       - echo "Building integration testing assemblies and gathering artifacts..."
       - ./testing/notification-test-aws/build-aws/prepare-dist.sh
 
-      - sed -i "s|__CHART_VERSION__|$VERSION|"  devops/aws/chart/Chart.yaml
       - sed -i "s|__CONTAINER__|$ECR_IMAGE|"  devops/aws/chart/values.yaml
       - echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--"
       - rsync -r devops/aws/* "${OUTPUT_DIR:-dist}"

provider/notification-aws/pom.xml

@@ -76,7 +76,13 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.21.0-rc1</version>
+            <version>0.21.0-rc5</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.11.0</version>
+            <scope>compile</scope>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-secretsmanager -->

provider/notification-aws/src/main/resources/application.properties

@@ -22,9 +22,6 @@ PARTITION_API=${PARTITION_BASE_URL}/api/partition/v1
 REGISTER_SERVICE_URL=${REGISTER_BASE_URL}/api/register/v1
 
 aws.ssm=${SSM_ENABLED:True}
-aws.environment=${RESOURCE_PREFIX}
-aws.parameter.prefix=/osdu/${RESOURCE_PREFIX}
-aws.primary.region=${aws.parameter.prefix}/primary-region
 
 app.expireTime=300
 app.maxCacheSize=10

testing/notification-test-aws/pom.xml

@@ -44,7 +44,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.14.0-rc2</version>
+            <version>0.21.0-rc5</version>
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>