Merge branch 'GONRG-9288-Update_core-plus_pipeline' into 'master'

Gonrg 9288 update core plus pipeline

See merge request !853

.gitlab-ci.yml

@@ -59,6 +59,9 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/gc-global.yml"
 
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/core-global.yml"
+
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/fossa-maven.yml"
 

devops/core-plus/deploy/Chart.yaml

+apiVersion: v2
+name: core-plus-storage-deploy
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.19.0"

devops/core-plus/deploy/README.md

+<!--- Deploy -->
+
+# Deploy helm chart
+
+## Introduction
+
+This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager.
+
+## Prerequisites
+
+The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6)
+> It is possible to use other versions, but it hasn't been tested
+
+### Operation system
+
+The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported.
+
+### Packages
+
+Packages are only needed for installation from a local computer.
+
+- **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/)
+- **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+
+## Installation
+
+First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
+
+### Global variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes
+**global.limitsEnabled** | whether CPU and memory limits are enabled | boolean | true | yes
+
+### Configmap variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**data.logLevel** | logging level | string | `ERROR` | yes
+**data.defaultDataCountry** | Data storage region | string | `US` | yes
+**data.storageServiceAccountEmail** | Storage service account email, used during OQM events processing | string | `storage@service.local` | yes
+**data.entitlementsHost** | Entitlements service host address | string | `http://entitlements` | yes
+**data.partitionHost** | Partition service host address | string | `http://partition` | yes
+**data.crsConverterHost** | CRS Converter service host address | string | `http://crs-conversion` | yes
+**data.legalHost** | Legal service host address | string | `http://legal` | yes
+**data.opaEnabled** | whether OPA is enabled | boolean | false | yes
+**data.opaEndpoint** | OPA host address | string | `http://opa` | yes
+**data.redisStorageHost** | The host for redis instance. If empty (by default), helm installs an internal redis instance | string | - | yes
+**data.redisStoragePort** | The port for redis instance | digit | 6379 | yes
+
+### Deployment variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**data.requestsCpu** | amount of requested CPU | string | `10m` | yes
+**data.requestsMemory** | amount of requested memory| string | `650Mi` | yes
+**data.limitsCpu** | CPU limit | string | `1` | only if `global.limitsEnabled` is true
+**data.limitsMemory** | memory limit | string | `3G` | only if `global.limitsEnabled` is true
+**data.image** | path to the image in a registry | string | - | yes
+**data.imagePullPolicy** | when to pull the image | string | `IfNotPresent` | yes
+**data.serviceAccountName** | name of kubernetes service account | string | `storage` | yes
+**data.redisImage** | service image | string | `redis:7` | yes
+
+### Configuration variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**conf.appName** | Service name | string | `storage` | yes
+**conf.keycloakSecretName** | secret for keycloak | string | `storage-keycloak-secret` | yes
+**conf.minioSecretName** | secret for minio | string | `storage-minio-secret` | yes
+**conf.postgresSecretName** | secret for postgres | string | `storage-postgres-secret` | yes
+**conf.rabbitmqSecretName** | secret for rabbitmq | string | `rabbitmq-secret` | yes
+**conf.storageRedisSecretName** | secret for redis that contains redis password with REDIS_PASSWORD key | string | `storage-redis-secret` | yes
+**conf.replicas** | Number of replicas | integer | 3 | yes
+
+### Istio variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**istio.proxyCPU** | CPU request for Envoy sidecars | string | 10m | yes
+**istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | 200m | yes
+**istio.proxyMemory** | memory request for Envoy sidecars | string | 100Mi | yes
+**istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | 256Mi | yes
+
+## Install the Helm chart
+
+Run this command from within this directory:
+
+```console
+helm install core-storage-deploy .
+```
+
+## Uninstall the Helm chart
+
+To uninstall the helm deployment:
+
+```console
+helm uninstall core-storage-deploy
+```
+
+> Do not forget to delete all k8s secrets and PVCs accociated with the Service.
+
+[Move-to-Top](#deploy-helm-chart)

devops/core-plus/deploy/templates/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ .Values.conf.appName | quote }}
+  name: {{ printf "%s-config" .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+data:
+  CRS_CONVERTER_HOST: {{ .Values.data.crsConverterHost | quote }}
+  DEFAULT_DATA_COUNTRY: {{ .Values.data.defaultDataCountry | quote }}
+  ENTITLEMENTS_HOST: {{ .Values.data.entitlementsHost | quote }}
+  LEGAL_HOST: {{ .Values.data.legalHost | quote }}
+  LOG_LEVEL: {{ .Values.data.logLevel | quote }}
+  PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
+  STORAGE_SERVICE_ACCOUNT_EMAIL: {{ .Values.data.storageServiceAccountEmail | quote }}
+  SPRING_PROFILES_ACTIVE: "antos"
+  OPA_ENABLED: {{ .Values.data.opaEnabled | quote }}
+  OPA_ENDPOINT: {{ .Values.data.opaEndpoint | quote }}
+  {{- if .Values.data.redisStorageHost }}
+  REDIS_GROUP_HOST: {{ .Values.data.redisStorageHost | quote }}
+  REDIS_STORAGE_HOST: {{ .Values.data.redisStorageHost | quote }}
+  {{- else }}
+  REDIS_GROUP_HOST: {{ printf "redis-%s" .Values.conf.appName | quote }}
+  REDIS_STORAGE_HOST: {{ printf "redis-%s" .Values.conf.appName | quote }}
+  {{- end }}
+  REDIS_STORAGE_PORT: {{ .Values.data.redisStoragePort | quote }}
+  REDIS_GROUP_PORT: {{ .Values.data.redisStoragePort | quote }}

devops/core-plus/deploy/templates/deploy-redis.yaml

+{{ if not .Values.data.redisStorageHost }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ printf "redis-%s" .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    type: infra
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ printf "redis-%s" .Values.conf.appName | quote }}
+  template:
+    metadata:
+      labels:
+        app: {{ printf "redis-%s" .Values.conf.appName | quote }}
+      annotations:
+        sidecar.istio.io/proxyCPU: 5m
+        sidecar.istio.io/proxyMemory: 64Mi
+    spec:
+      containers:
+      - args:
+        - --requirepass
+        - $(REDIS_PASSWORD)
+        envFrom:
+        - secretRef:
+            name: {{ .Values.conf.storageRedisSecretName | quote }}
+        image: {{ .Values.data.redisImage | quote }}
+        imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }}
+        name: {{ printf "redis-%s" .Values.conf.appName | quote }}
+        ports:
+        - containerPort: 6379
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 5m
+            memory: 20Mi
+{{ end }}

devops/core-plus/deploy/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{ .Values.conf.appName | quote }}
+    type: core
+    source: java
+  name: {{ .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.conf.appName | quote }}
+  replicas: {{ .Values.conf.replicas }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.conf.appName | quote }}
+      annotations:
+        rollme: {{ randAlphaNum 5 | quote }}
+        sidecar.istio.io/proxyCPU: {{ .Values.istio.proxyCPU | quote }}
+        sidecar.istio.io/proxyMemory: {{ .Values.istio.proxyMemory | quote }}
+        sidecar.istio.io/proxyCPULimit: {{ .Values.istio.proxyCPULimit | quote }}
+        sidecar.istio.io/proxyMemoryLimit: {{ .Values.istio.proxyMemoryLimit | quote }}
+    spec:
+      containers:
+      - name: {{ .Values.conf.appName | quote }}
+        image: {{ .Values.data.image | quote }}
+        imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }}
+        env:
+        - name: REDIS_GROUP_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.conf.storageRedisSecretName | quote }}
+              key: REDIS_PASSWORD
+        - name: REDIS_STORAGE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.conf.storageRedisSecretName | quote }}
+              key: REDIS_PASSWORD
+        envFrom:
+        - configMapRef:
+            name: {{ printf "%s-config" .Values.conf.appName | quote }}
+        - secretRef:
+            name: {{ .Values.conf.keycloakSecretName | quote }}
+        - secretRef:
+            name: {{ .Values.conf.minioSecretName | quote }}
+        - secretRef:
+            name: {{ .Values.conf.postgresSecretName | quote }}
+        - secretRef:
+            name: {{ .Values.conf.rabbitmqSecretName | quote }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+        ports:
+        - containerPort: 8080
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /api/storage/v2/liveness_check
+            port: 8080
+          initialDelaySeconds: 120
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: {{ .Values.data.requestsCpu | quote }}
+            memory: {{ .Values.data.requestsMemory | quote }}
+          {{- if .Values.global.limitsEnabled }}
+          limits:
+            cpu: {{ .Values.data.limitsCpu | quote }}
+            memory: {{ .Values.data.limitsMemory | quote }}
+          {{- end }}
+      serviceAccountName: {{ .Values.data.serviceAccountName | quote }}

devops/core-plus/deploy/templates/service-account.yaml

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.data.serviceAccountName | quote }}
+  namespace: {{ .Release.Namespace | quote }}

devops/core-plus/deploy/templates/service-redis.yaml

+{{ if not .Values.data.redisStorageHost }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "redis-%s" .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  ports:
+  - port: {{ .Values.data.redisStoragePort }}
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: {{ printf "redis-%s" .Values.conf.appName | quote }}
+{{ end }}

devops/core-plus/deploy/templates/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    cloud.google.com/neg: '{"ingress": true}'
+  name: {{ .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+      name: http
+  selector:
+    app: {{ .Values.conf.appName | quote }}

devops/core-plus/deploy/templates/virtual-service.yaml

+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: {{ .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  hosts:
+    - {{ printf "osdu.%s" .Values.global.domain | quote }}
+  gateways:
+    - service-gateway
+  http:
+    - match:
+        - uri:
+            prefix: "/api/storage"
+      route:
+        - destination:
+            port:
+              number: 80
+            host: {{ printf "%s.%s.svc.cluster.local" .Values.conf.appName .Release.Namespace | quote }}

devops/core-plus/deploy/values.yaml

+# Default values for storage-deploy.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+global:
+  domain: ""
+  limitsEnabled: true
+
+data:
+  # configmaps
+  logLevel: "ERROR"
+  defaultDataCountry: "US"
+  storageServiceAccountEmail: "storage@service.local"
+  entitlementsHost: "http://entitlements"
+  partitionHost: "http://partition"
+  crsConverterHost: "http://crs-conversion"
+  legalHost: "http://legal"
+  opaEnabled: false
+  opaEndpoint: "http://opa"
+  # deployments
+  requestsCpu: "7m"
+  requestsMemory: "450Mi"
+  limitsCpu: "1"
+  limitsMemory: "3G"
+  image: ""
+  imagePullPolicy: "IfNotPresent"
+  serviceAccountName: "storage"
+  # redis connection. Change it if you want to use external redis
+  redisStorageHost: ""
+  redisStoragePort: 6379
+  redisImage: "docker.io/library/redis:7"
+
+conf:
+  appName: "storage"
+  keycloakSecretName: "storage-keycloak-secret"
+  minioSecretName: "storage-minio-secret"
+  postgresSecretName: "storage-postgres-secret"
+  rabbitmqSecretName: "rabbitmq-secret"
+  replicas: 3
+  storageRedisSecretName: "storage-redis-secret"
+
+istio:
+  proxyCPU: "5m"
+  proxyCPULimit: "200m"
+  proxyMemory: "50Mi"
+  proxyMemoryLimit: "256Mi"

devops/core-plus/pipeline/override-stages.yml

-gc-containerize-gitlab:core-plus:
-  stage: containerize
-  needs:
-    - "compile-and-unit-test"
-    - "download_plugins"
-  tags: ["osdu-small"]
-  image: docker:19.03
-  cache: {}
-  variables:
-    BUILD_ARGS: "--build-arg PORT=$GC_PORT"
-    BUILD_PATH: "storage-core-plus/cloudbuild/Dockerfile"
-    IMAGE_NAME: "storage-core-plus"
-  script:
-    - !reference [.gc_set_image_name, script]
-    - docker build $EXTRA_TAG -f $BUILD_PATH $BUILD_ARGS .
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME
-  rules:
-    - if: "$CI_COMMIT_BRANCH =~ /^release/"
-      when: never
-    - if: "$CI_COMMIT_TAG"
-      when: never
-    - if: "$PROTECTED == '1'"
-      when: on_success
-
-gc-helm-charts-gitlab:core-plus:
-  stage: containerize
-  tags: ["osdu-small"]
-  needs: ["gc-containerize-gitlab:core-plus"]
-  image:
-    name: alpine/helm:3.11.2
-    entrypoint: [""]
-  variables:
-    IMAGE_NAME: "storage-core-plus"
-  before_script:
-    - helm registry login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - !reference [.gc_set_image_name, script]
-    - !reference [.gc_substitute_image_in_helm, script]
-  script:
-    - !reference [.gc_define_app_version, script]
-    - helm package $GC_HELM_DEPLOYMENT_DIR -u --version $GC_HELM_PACKAGE_VERSION-$HELM_TAG --app-version $APP_VERSION
-    - helm push $GC_HELM_PACKAGE_NAME-$GC_HELM_PACKAGE_VERSION-$HELM_TAG.tgz oci://$CI_REGISTRY_IMAGE/core-plus-helm
-  rules:
-    - if: "$CI_COMMIT_BRANCH =~ /^release/"
-      when: never
-    - if: "$CI_COMMIT_TAG"
-      when: never
-    - if: "$PROTECTED == '1'"
-      when: on_success
+variables:
+  CORE_SERVICE: storage-core-plus
+  CORE_BUILD_PATH: "storage-core-plus/cloudbuild/Dockerfile"
+  CORE_HELM_PACKAGE_NAME: core-plus-storage-deploy
+  CORE_HELM_TIMEOUT: "--timeout 15m"
+  OPA_INTEGRATION_ENABLED: "true"
+  CORE_TEST_PROJECT: "storage-test-core,storage-test-baremetal"
+  CORE_TEST_SUBDIR: "testing/storage-test-baremetal"
 
-gc-baremetal-deploy:
-  environment:
-    name: GC_Baremetal
-  extends: .gc-baremetal-variables
-  id_tokens:
-    GITLAB_OIDC_TOKEN:
-      aud: https://iam.googleapis.com/projects/${GC_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GC_POOL_ID}/providers/${GC_PROVIDER_ID}
-  tags: ["osdu-small"]
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
-  stage: deploy
-  needs:
-    - "gc-containerize-gitlab:core-plus"
-    - "gc-helm-charts-gitlab:core-plus"
-  retry: 1
+core-deploy:
   variables:
-    IMAGE_NAME: "storage-core-plus"
-    GC_SA_EMAIL: $GC_SA_GKE_EMAIL
-  script:
-    - !reference [.gc_obtain_credentials, script]
-    - !reference [.gc_set_image_name, script]
-    - !reference [.gc_common_config, script]
-    - >
-      helm upgrade $GC_SERVICE-deploy oci://$CI_REGISTRY_IMAGE/core-plus-helm/$GC_HELM_PACKAGE_NAME
-      --version $GC_HELM_PACKAGE_VERSION-$HELM_TAG
-      --install
-      --create-namespace
-      --namespace=$GC_HELM_NAMESPACE
-      --wait
-      --history-max=3
-      --set global.onPremEnabled=true
-      --set global.domain=$GC_DOMAIN
-      --set data.serviceAccountName=$GC_SERVICE
-      --set data.bootstrapServiceAccountName=$GC_BOOTSTRAP_SA
-      --set data.logLevel=INFO
-      --set data.springProfilesActive=$GC_SPRING_PROFILES_ACTIVE
-      --set data.bucketPrefix=$BUCKET_PREFIX
-      --set data.dataPartitionId=$GC_TENANT
-      --set data.groupId=$GROUP_ID
-      --set data.adminUserEmail=$GC_ADMIN_USER_EMAIL
-      --set data.airflowComposerEmail=$GC_AIRFLOW_COMPOSER_EMAIL
-      --set data.sharedTenantName=$GC_TENANT
-      --set opa.data.serviceAccountName=opa
-      --set data.bucketName=$GC_POLICY_BUCKET
-      --set data.googleCloudProject=$GC_PROJECT
-      --set data.subscriberPrivateKeyId=$GC_BAREMETAL_SUBSCRIBER_PRIVATE_KEY_ID
-      $GC_HELM_SETS
-      $GC_HELM_TIMEOUT
-    - !reference [.gc_verify_deploy, script]
-    - !reference [.gc_verify_bootstrap, script]
-
-  rules:
-    - if: "$CI_COMMIT_BRANCH =~ /^release/"
-      when: never
-    - if: "$CI_COMMIT_TAG"
-      when: never
-    - if: "$PROTECTED == '1'"
-      when: on_success
+    CORE_SERVICE: "storage"
 
-gc-baremetal-test:
+core-test:
   variables:
-    GC_VENDOR: baremetal
-    DOMAIN: $GROUP_ID
+    OPA_INTEGRATION_ENABLED: "false"