diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7b0fb37dba40b0947b76b3f3a92ad9b5f422e62..173fd3f655218e072e1671890075ccdbca5f8d8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,9 @@ variables: IBM_HELM_CONFIG_PATH: devops/ibm/ibm-partition-config IBM_HELM_DEPLOY_PATH: devops/ibm/ibm-partition-deploy + # FIXME remove when all services are migrated to a single helm + OSDU_GCP_ENABLE_HELM_CONFIG: "false" + include: - project: "osdu/platform/ci-cd-pipelines" file: "standard-setup.yml" diff --git a/devops/gcp/configmap/Chart.yaml b/devops/gcp/configmap/Chart.yaml deleted file mode 100644 index 75d7fdd9463f1d1b460f5c8aa94aa9d701edd34c..0000000000000000000000000000000000000000 --- a/devops/gcp/configmap/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v2 -name: gcp-partition-configmap -description: A Helm chart for Kubernetes -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.16.0 diff --git a/devops/gcp/configmap/templates/configmap.yml b/devops/gcp/configmap/templates/configmap.yml deleted file mode 100644 index 77ad50dc6788bdd905f2d09b32115a6a32d22d75..0000000000000000000000000000000000000000 --- a/devops/gcp/configmap/templates/configmap.yml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app: "{{ .Values.conf.appName }}" - name: "{{ .Values.conf.configmap }}" - namespace: "{{ .Release.Namespace }}" -data: - LOG_LEVEL: "{{ .Values.data.logLevel }}" - SPRING_PROFILES_ACTIVE: "{{ .Values.data.springProfilesActive }}" - {{- if not .Values.conf.onPremEnabled }} - GOOGLE_CLOUD_PROJECT: "{{ .Values.data.projectId }}" - PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partitionAdminAccounts }}" - GOOGLE_AUDIENCES: "{{ .Values.data.googleAudiences }}" - SERVICE_ACCOUNT_TAIL: "{{ .Values.data.serviceAccountTail }}" - {{- end }} diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml deleted file mode 100644 index 0268164a1eaec425d4cb2d1261a4f00de295bd60..0000000000000000000000000000000000000000 --- a/devops/gcp/configmap/values.yaml +++ /dev/null @@ -1,23 +0,0 @@ -data: - # common - logLevel: "ERROR" - springProfilesActive: "gcp" - # gcp - projectId: "" - dataProjectId: "" - partitionAdminAccounts: "" - googleAudiences: "" - serviceAccountTail: ".iam.gserviceaccount.com" - # bootstrap common - partitionName: "partition" - dataPartitionId: "" - dataPartitionIdList: [] - datafierSa: "datafier" - partitionCleanUpEnabled: "false" - # bootstrap onprem - domain: "" - minioExternalEndpoint: "" # use only if external minio is configured -conf: - configmap: "partition-config" - appName: "partition" - onPremEnabled: false diff --git a/devops/gcp/deploy/README.md b/devops/gcp/deploy/README.md index 37abd4cb7b83c526f3aade6696ea65ab62471231..a40361d54dc70a7adbc5ecb1f34829b758535da6 100644 --- a/devops/gcp/deploy/README.md +++ b/devops/gcp/deploy/README.md @@ -92,37 +92,56 @@ Packages are only needed for installation from a local computer. First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. -### Common variables +### Configmap variables | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| -**image** | your image name | string | - | yes -**requestsCpu** | amount of requests CPU | string | 0.1 | yes -**requestsMemory** | amount of requests memory| string | 260M | yes -**limitsCpu** | CPU limit | string | 1 | yes -**limitsMemory** | memory limit | string | 1G | yes -**serviceAccountName** | name of your service account | string | partition | yes -**imagePullPolicy** | when to pull image | string | IfNotPresent | yes +**logLevel** | logging level | string | INFO | yes +**springProfilesActive** | active spring profile | string | gcp | yes +**projectId** | your GCP project id | string | - | yes +**dataProjectId** | data GCP project id (for pipelines) | string | - | no +**partitionAdminAccounts** | admin accounts validated by partition service | string | - | yes +**googleAudiences** | your GCP client id | string | - | yes +**serviceAccountTail** | ending of GCP service account | string | .iam.gserviceaccount.com | yes +**partitionName** | partition host | string | partition | yes +**dataPartitionId** | data partition id | string | - | yes +**datafierSa** | datafier service account | string | datafier | yes +**minioExternalEndpoint** | api url for external minio, if external minio is configured - this value will be set for MINIO_ENDPOINT and FILE_MINIO_ENDPOINT in bootstrap configmap | string | - | no + +> googleAudiences: If you are logged in to GCP console with `gcloud auth application-default login --no-browser` from your terminal, you can get your client_id using the command: -### Bootstrap variables +```console +cat ~/.config/gcloud/application_default_credentials.json | grep client_id +``` + +### Deployment variables | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| +**requestsCpu** | amount of requests CPU | string | 0.1 | yes +**requestsMemory** | amount of requests memory | string | 260M | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1G | yes +**serviceAccountName** | name of your service account | string | partition | yes +**image** | path to the image in a registry | string | - | yes +**imagePullPolicy** | when to pull the image | string | IfNotPresent | yes **bootstrapImage** | name of the bootstrap image | string | - | yes -**bootstrapServiceAccountName** | name of the bootstrap SA | string | - | yes +**bootstrapServiceAccountName** | name of the bootstrap service account | string | - | yes -### Config variables +### Configuration variables | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| -**domain** | your domain | string | - | yes **appName** | name of the app | string | partition | yes **cicdEnabled** | whether CI/CD is enabled | boolean | false | yes -**istioEnabled** | whether to enable istio resources | boolean | true | yes **configmap** | configmap to be used | string | partition-config | yes +**domain** | your domain | string | - | yes +**istioEnabled** | whether to enable istio resources | boolean | true | yes **onPremEnabled** | whether on-prem is enabled | boolean | false | yes **publicAvailable** | public access to /api/partition | boolean | false | yes **secret** | secret for postgres | string | partition-postgres-secret | yes +**mtlsMode** | MTLS mode | string | STRICT | yes +**realm** | realm in keycloak | string | osdu | yes ### Install the helm chart diff --git a/devops/gcp/deploy/templates/authorization-policy.yml b/devops/gcp/deploy/templates/authorization-policy.yaml similarity index 94% rename from devops/gcp/deploy/templates/authorization-policy.yml rename to devops/gcp/deploy/templates/authorization-policy.yaml index 5f35912c0249614ac7cc242df6589ae207f4eebd..6334d671298f99f51c3e96ef9cbf0fda11daf5c2 100644 --- a/devops/gcp/deploy/templates/authorization-policy.yml +++ b/devops/gcp/deploy/templates/authorization-policy.yaml @@ -3,12 +3,12 @@ apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: "{{ .Values.conf.appName }}-jwt-policy" - namespace: "{{ .Release.Namespace }}" + name: {{ printf "%s-jwt-policy" .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} spec: selector: matchLabels: - app: "{{ .Values.conf.appName }}" + app: {{ .Values.conf.appName | quote }} action: ALLOW rules: - from: diff --git a/devops/gcp/configmap/templates/configmap-bootstrap.yml b/devops/gcp/deploy/templates/configmap-bootstrap.yaml similarity index 92% rename from devops/gcp/configmap/templates/configmap-bootstrap.yml rename to devops/gcp/deploy/templates/configmap-bootstrap.yaml index fba1927b09c1013093e5368de972ff1eee733ca4..868778dde1858f50c10140d94ad1df3ef43261dc 100644 --- a/devops/gcp/configmap/templates/configmap-bootstrap.yml +++ b/devops/gcp/deploy/templates/configmap-bootstrap.yaml @@ -17,9 +17,9 @@ data: {{- end }} {{- if .Values.conf.onPremEnabled }} SERVICE_ACCOUNT: {{ printf "%s@service.local" .Values.data.datafierSa | quote }} - DOMAIN: "{{ .Values.data.domain }}" + DOMAIN: "{{ .Values.conf.domain }}" MINIO_ENDPOINT: {{ default "http://minio:9000" .Values.data.minioExternalEndpoint | quote }} - FILE_MINIO_ENDPOINT: {{ default (printf "https://s3.%s" .Values.data.domain) .Values.data.minioExternalEndpoint | quote }} + FILE_MINIO_ENDPOINT: {{ default (printf "https://s3.%s" .Values.conf.domain) .Values.data.minioExternalEndpoint | quote }} {{- else }} SERVICE_ACCOUNT: {{ printf "%s@%s.iam.gserviceaccount.com" .Values.data.datafierSa .Values.data.projectId | quote }} AUDIENCES: "{{ .Values.data.googleAudiences }}" diff --git a/devops/gcp/deploy/templates/configmap.yaml b/devops/gcp/deploy/templates/configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f364a260be023ec82fd99fc366be784576a0fac2 --- /dev/null +++ b/devops/gcp/deploy/templates/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: {{ .Values.conf.appName | quote }} + name: {{ .Values.conf.configmap | quote }} + namespace: {{ .Release.Namespace | quote }} +data: + LOG_LEVEL: {{ .Values.data.logLevel | quote }} + SPRING_PROFILES_ACTIVE: {{ .Values.data.springProfilesActive | quote }} + {{- if not .Values.conf.onPremEnabled }} + GOOGLE_CLOUD_PROJECT: {{ .Values.data.projectId | quote }} + PARTITION_ADMIN_ACCOUNTS: {{ .Values.data.partitionAdminAccounts | quote }} + {{- end }} diff --git a/devops/gcp/deploy/templates/deploy-bootstrap.yaml b/devops/gcp/deploy/templates/deploy-bootstrap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daeb5bcd7eb173f22de5e3eed362ff4f7f75642b --- /dev/null +++ b/devops/gcp/deploy/templates/deploy-bootstrap.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-bootstrap" .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }} + template: + metadata: + labels: + app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }} + annotations: + rollme: {{ randAlphaNum 5 | quote }} + spec: + containers: + - image: {{ .Values.data.bootstrapImage | quote }} + imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }} + name: {{ printf "%s-bootstrap" .Values.conf.appName | quote }} + readinessProbe: + exec: + command: + - cat + - /tmp/bootstrap_ready + envFrom: + - configMapRef: + name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }} + restartPolicy: Always + serviceAccountName: {{ .Values.data.bootstrapServiceAccountName | quote }} diff --git a/devops/gcp/deploy/templates/deploy-bootstrap.yml b/devops/gcp/deploy/templates/deploy-bootstrap.yml deleted file mode 100644 index 57486741ea6c494f1bafd7f5cbd8e41ca96784c5..0000000000000000000000000000000000000000 --- a/devops/gcp/deploy/templates/deploy-bootstrap.yml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "{{ .Values.conf.appName }}-bootstrap" - namespace: "{{ .Release.Namespace }}" -spec: - replicas: 1 - selector: - matchLabels: - app: "{{ .Values.conf.appName }}-bootstrap" - template: - metadata: - labels: - app: "{{ .Values.conf.appName }}-bootstrap" - annotations: - rollme: {{ randAlphaNum 5 | quote }} - spec: - containers: - - image: "{{ .Values.data.bootstrapImage }}" - imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" - name: "{{ .Values.conf.appName }}-bootstrap" - readinessProbe: - exec: - command: - - cat - - /tmp/bootstrap_ready - envFrom: - - configMapRef: - name: "{{ .Values.conf.configmap }}-bootstrap" - restartPolicy: Always - serviceAccountName: "{{ .Values.data.bootstrapServiceAccountName }}" diff --git a/devops/gcp/deploy/templates/deploy.yaml b/devops/gcp/deploy/templates/deploy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7b87522ef4bc32d2a755eb0f25cd187cade639c --- /dev/null +++ b/devops/gcp/deploy/templates/deploy.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.conf.appName | quote }} + template: + metadata: + labels: + app: {{ .Values.conf.appName | quote }} + annotations: + rollme: {{ randAlphaNum 5 | quote }} + spec: + containers: + - image: {{ .Values.data.image | quote }} + imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }} + name: {{ .Values.conf.appName | quote }} + envFrom: + - configMapRef: + name: {{ .Values.conf.configmap | quote }} + {{- if .Values.conf.onPremEnabled }} + - secretRef: + name: {{ .Values.conf.secret | quote }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + ports: + - containerPort: 8080 + resources: + requests: + cpu: {{ .Values.data.requestsCpu | quote }} + memory: {{ .Values.data.requestsMemory | quote }} + limits: + cpu: {{ .Values.data.limitsCpu | quote }} + memory: {{ .Values.data.limitsMemory | quote }} + serviceAccountName: {{ .Values.data.serviceAccountName | quote }} diff --git a/devops/gcp/deploy/templates/deploy.yml b/devops/gcp/deploy/templates/deploy.yml deleted file mode 100644 index f7136812751771f2c4fb80f876748e63f2115bf8..0000000000000000000000000000000000000000 --- a/devops/gcp/deploy/templates/deploy.yml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "{{ .Values.conf.appName }}" - namespace: "{{ .Release.Namespace }}" -spec: - replicas: 1 - selector: - matchLabels: - app: "{{ .Values.conf.appName }}" - template: - metadata: - labels: - app: "{{ .Values.conf.appName }}" - annotations: - rollme: {{ randAlphaNum 5 | quote }} - spec: - containers: - - image: "{{ .Values.data.image }}" - imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" - name: "{{ .Values.conf.appName }}" - envFrom: - - configMapRef: - name: "{{ .Values.conf.configmap }}" - {{- if .Values.conf.onPremEnabled }} - - secretRef: - name: "{{ .Values.conf.secret }}" - {{- end }} - securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - ports: - - containerPort: 8080 - resources: - requests: - cpu: "{{ .Values.data.requestsCpu }}" - memory: "{{ .Values.data.requestsMemory }}" - limits: - cpu: "{{ .Values.data.limitsCpu }}" - memory: "{{ .Values.data.limitsMemory }}" - serviceAccountName: "{{ .Values.data.serviceAccountName }}" diff --git a/devops/gcp/deploy/templates/peer-authentication.yml b/devops/gcp/deploy/templates/peer-authentication.yaml similarity index 100% rename from devops/gcp/deploy/templates/peer-authentication.yml rename to devops/gcp/deploy/templates/peer-authentication.yaml diff --git a/devops/gcp/deploy/templates/request-authentication.yml b/devops/gcp/deploy/templates/request-authentication.yaml similarity index 86% rename from devops/gcp/deploy/templates/request-authentication.yml rename to devops/gcp/deploy/templates/request-authentication.yaml index 0c3f4c7203aeb61bbb6b87e684c9e0fb9b4a8d0a..918e097c2e485a434615dec80a4dc0c27cf4ef3e 100644 --- a/devops/gcp/deploy/templates/request-authentication.yml +++ b/devops/gcp/deploy/templates/request-authentication.yaml @@ -3,12 +3,12 @@ apiVersion: security.istio.io/v1beta1 kind: RequestAuthentication metadata: - name: "{{ .Values.conf.appName }}-jwt-policy" - namespace: "{{ .Release.Namespace }}" + name: {{ printf "%s-jwt-policy" .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} spec: selector: matchLabels: - app: "{{ .Values.conf.appName }}" + app: {{ .Values.conf.appName | quote }} jwtRules: - issuer: "https://keycloak.{{ .Values.conf.domain }}/realms/{{ .Values.auth.realm }}" jwksUri: "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/realms/{{ .Values.auth.realm }}/protocol/openid-connect/certs" diff --git a/devops/gcp/deploy/templates/service-account.yaml b/devops/gcp/deploy/templates/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0771985cd5da9db77781dc3ab635e52378b3fb6 --- /dev/null +++ b/devops/gcp/deploy/templates/service-account.yaml @@ -0,0 +1,7 @@ +{{- if .Values.conf.onPremEnabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.data.serviceAccountName | quote }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/devops/gcp/deploy/templates/service-account.yml b/devops/gcp/deploy/templates/service-account.yml deleted file mode 100644 index 25fd74efbfca059f541af82d98b676cfcb930d64..0000000000000000000000000000000000000000 --- a/devops/gcp/deploy/templates/service-account.yml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if .Values.conf.onPremEnabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "{{ .Values.data.serviceAccountName }}" - namespace: "{{ .Release.Namespace }}" -{{- end }} diff --git a/devops/gcp/deploy/templates/service.yml b/devops/gcp/deploy/templates/service.yaml similarity index 65% rename from devops/gcp/deploy/templates/service.yml rename to devops/gcp/deploy/templates/service.yaml index 5212e9349ffaebe0cdf4e6e3615ebe07717dc484..fdfe296cf79f7533f42c8f0f494d87a44cee32cb 100644 --- a/devops/gcp/deploy/templates/service.yml +++ b/devops/gcp/deploy/templates/service.yaml @@ -1,8 +1,8 @@ apiVersion: v1 kind: Service metadata: - name: "{{ .Values.conf.appName }}" - namespace: "{{ .Release.Namespace }}" + name: {{ .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} {{- if not .Values.conf.onPremEnabled }} annotations: cloud.google.com/neg: '{"ingress": true}' @@ -14,4 +14,4 @@ spec: protocol: TCP name: http selector: - app: "{{ .Values.conf.appName }}" + app: {{ .Values.conf.appName | quote }} diff --git a/devops/gcp/deploy/templates/virtual-service.yml b/devops/gcp/deploy/templates/virtual-service.yaml similarity index 62% rename from devops/gcp/deploy/templates/virtual-service.yml rename to devops/gcp/deploy/templates/virtual-service.yaml index e5da74f22f51bacdbb25d701fdcce2f4c77974e0..90def2295d6faa08fddc32252634eebe46f1a2ec 100644 --- a/devops/gcp/deploy/templates/virtual-service.yml +++ b/devops/gcp/deploy/templates/virtual-service.yaml @@ -2,12 +2,14 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.appName }}" - namespace: "{{ .Release.Namespace }}" + name: {{ .Values.conf.appName | quote }} + namespace: {{ .Release.Namespace | quote }} spec: hosts: - {{- if .Values.conf.domain }} + {{- if and .Values.conf.domain .Values.conf.onPremEnabled }} - {{ printf "osdu.%s" .Values.conf.domain | quote }} + {{- else if .Values.conf.domain }} + - {{ .Values.conf.domain | quote }} {{- else }} - "*" {{- end }} @@ -27,5 +29,5 @@ spec: - destination: port: number: 80 - host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" + host: {{ printf "%s.%s.svc.cluster.local" .Values.conf.appName .Release.Namespace | quote }} {{- end }} diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index 52cdb52da41be07d12de367ad2538227fd6573c2..701897cdb4ee46ddf4acfc7e2d7e41f18bb65eb5 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -1,4 +1,18 @@ data: + # configmaps + logLevel: "ERROR" + springProfilesActive: "gcp" + projectId: "" + googleAudiences: "" + dataProjectId: "" + dataPartitionId: "" + dataPartitionIdList: [] + partitionAdminAccounts: "" + partitionName: "partition" + datafierSa: "datafier" + partitionCleanUpEnabled: "false" + minioExternalEndpoint: "" # use only if external minio is configured + # deployments requestsCpu: "0.1" requestsMemory: "260M" limitsCpu: "1" @@ -6,19 +20,19 @@ data: serviceAccountName: "partition" imagePullPolicy: "IfNotPresent" image: "" - # bootstrap bootstrapImage: "" bootstrapServiceAccountName: "" + # bootstrap onprem conf: appName: "partition" cicdEnabled: false - istioEnabled: true configmap: "partition-config" domain: "" + istioEnabled: true onPremEnabled: false publicAvailable: false secret: "partition-postgres-secret" namespacePolicy: - mtlsMode: STRICT + mtlsMode: STRICT auth: - realm: "osdu" + realm: "osdu"