Skip to content
Snippets Groups Projects
Commit fc0ff54f authored by Marc Burnie [AWS]'s avatar Marc Burnie [AWS]
Browse files

Merge branch 'master' into master-dev-merge

parents 6a872bb3 49c59f54
No related branches found
No related tags found
2 merge requests!229Merge branch 'dependency-upgrade' into 'master',!195Use MongoPropertiesReader from os-core-lib-aws and add maxIdelTimeMS setting
Showing
with 137 additions and 121 deletions
......@@ -13,20 +13,30 @@ variables:
IBM_BUILD_SUBDIR: provider/partition-ibm
IBM_INT_TEST_SUBDIR: testing/partition-test-ibm
# --- osdu gcp specific variables ---
OSDU_GCP_ENABLE_BOOTSTRAP: "true"
OSDU_GCP_SERVICE: partition
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-partition
MAVEN_PROJECTS: "-pl partition-core,provider/partition-gcp"
OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_NAMESPACE: default
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.partition_admin_accounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS --set data.project_id=$OSDU_GCP_PROJECT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.log_level=INFO --set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.kms_key=$OSDU_GCP_PARTITION_KMS_KEY --set data.data_partition_id=$OSDU_GCP_TENANT"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.partition_admin_accounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS_DEV2 --set data.project_id=$OSDU_GCP_PROJECT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.log_level=INFO --set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.kms_key=$OSDU_GCP_PARTITION_KMS_KEY --set data.data_partition_id=$OSDU_GCP_TENANT"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.bootstrap_image=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-partition:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
--set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
--set data.projectId=$OSDU_GCP_PROJECT
--set data.googleAudiences=$GOOGLE_AUDIENCE
--set data.logLevel=INFO
--set data.dataPartitionId=$OSDU_GCP_TENANT
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
--set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
--set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
--set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
--set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS_DEV2
--set data.projectId=$OSDU_GCP_PROJECT
--set data.googleAudiences=$GOOGLE_AUDIENCE
--set data.logLevel=INFO
--set data.dataPartitionId=$OSDU_GCP_TENANT
# FIXME add value below to DEV2 pipeline
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: "--set data.bootstrap_image=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-partition:$CI_COMMIT_SHORT_SHA --set data.bootstrapServiceAccountName=$OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: >
--set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
--set data.bootstrapServiceAccountName=$OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT
OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
......@@ -62,30 +72,38 @@ include:
file: "publishing/pages.yml"
osdu-gcp-deploy-deployment:
variables:
OSDU_GCP_BOOTSTRAP_SERVICE: partition-bootstrap
needs: ["osdu-gcp-containerize-gitlab", "osdu-gcp-containerize-bootstrap-gitlab", "osdu-gcp-deploy-configmap"]
needs:
- osdu-gcp-containerize-gitlab
- osdu-gcp-containerize-bootstrap-gitlab
- osdu-gcp-deploy-configmap
after_script:
- echo ----- Verify Bootstrap -----
- kubectl rollout status deployment.v1.apps/$OSDU_GCP_BOOTSTRAP_SERVICE -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
- POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_BOOTSTRAP_SERVICE | tail -1 | awk '{print $1}')
- kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
- POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
- STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
- echo $STATUS
- if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
osdu-gcp-dev2-deploy-deployment:
variables:
OSDU_GCP_BOOTSTRAP_SERVICE: partition-bootstrap
OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT: workload-gke-bootstrap-sa
needs: ["osdu-gcp-containerize-gitlab", "osdu-gcp-containerize-bootstrap-gitlab", "osdu-gcp-dev2-deploy-configmap"]
needs:
- osdu-gcp-containerize-gitlab
- osdu-gcp-containerize-bootstrap-gitlab
- osdu-gcp-dev2-deploy-configmap
after_script:
- echo ----- Verify Bootstrap -----
- kubectl rollout status deployment.v1.apps/$OSDU_GCP_BOOTSTRAP_SERVICE -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
- POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_BOOTSTRAP_SERVICE | tail -1 | awk '{print $1}')
- kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
- POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
- STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
- echo $STATUS
- if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
osdu-gcp-anthos-deploy-deployment:
needs:
- osdu-gcp-containerize-gitlab
- osdu-gcp-containerize-bootstrap-gitlab
- osdu-gcp-anthos-deploy-configmap
osdu-gcp-test:
variables:
......
......@@ -2,18 +2,18 @@ apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: "{{ .Values.conf.bootstrap_name }}"
name: "{{ .Values.conf.bootstrap_configmap }}"
app: "{{ .Values.conf.appName }}-bootstrap"
name: "{{ .Values.conf.configmap }}-bootstrap"
namespace: "{{ .Release.Namespace }}"
data:
PARTITION_NAME: "{{ .Values.data.partition_name }}.{{ .Release.Namespace }}.svc.cluster.local"
PROJECT_ID: "{{ .Values.data.project_id }}"
DATA_PARTITION_ID: "{{ .Values.data.data_partition_id }}"
{{- if .Values.conf.on_prem_enabled }}
DOMAIN: "{{ .Values.data.domain }}"
PARTITION_NAME: "{{ .Values.data.partitionName }}.{{ .Release.Namespace }}.svc.cluster.local"
PROJECT_ID: "{{ .Values.data.projectId }}"
DATA_PARTITION_ID: "{{ .Values.data.dataPartitionId }}"
{{- if .Values.conf.onPremEnabled }}
DOMAIN: "{{- default (printf "%s.%s" ((index (lookup "v1" "Service" .Values.data.istioNamespace "istio-ingressgateway").status.loadBalancer.ingress 0).ip) "nip.io") .Values.data.domain -}}"
ENVIRONMENT: "{{ .Values.data.environment }}"
CLIENT_ID: "{{ .Values.data.client_id }}"
CLIENT_ID: "{{ .Values.data.clientId }}"
{{- else }}
AUDIENCES: "{{ .Values.data.google_audiences }}"
DATAFIER_SA: "{{ .Values.data.datafier_sa }}"
AUDIENCES: "{{ .Values.data.googleAudiences }}"
DATAFIER_SA: "{{ .Values.data.datafierSa }}"
{{- end }}
......@@ -2,18 +2,17 @@ apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: "{{ .Values.conf.app_name }}"
app: "{{ .Values.conf.appName }}"
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
LOG_LEVEL: "{{ .Values.data.log_level }}"
SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
{{- if .Values.conf.on_prem_enabled }}
PARTITION_PROPERTY_KIND: "{{ .Values.data.partition_property_kind }}"
PARTITION_NAMESPACE: "{{ .Values.data.partition_namespace }}"
LOG_LEVEL: "{{ .Values.data.logLevel }}"
SPRING_PROFILES_ACTIVE: "{{ .Values.data.springProfilesActive }}"
{{- if .Values.conf.onPremEnabled }}
PARTITION_NAMESPACE: "{{ .Values.data.partitionNamespace }}"
{{- else }}
GOOGLE_CLOUD_PROJECT: "{{ .Values.data.project_id }}"
PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partition_admin_accounts }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
SERVICE_ACCOUNT_TAIL: "{{ .Values.data.service_account_tail }}"
GOOGLE_CLOUD_PROJECT: "{{ .Values.data.projectId }}"
PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partitionAdminAccounts }}"
GOOGLE_AUDIENCES: "{{ .Values.data.googleAudiences }}"
SERVICE_ACCOUNT_TAIL: "{{ .Values.data.serviceAccountTail }}"
{{- end }}
data:
project_id: ""
partition_admin_accounts: ""
google_audiences: ""
log_level: "INFO"
service_account_tail: ".iam.gserviceaccount.com"
partition_property_kind: ""
partition_namespace: ""
spring_profiles_active: "gcp"
projectId: ""
partitionAdminAccounts: ""
googleAudiences: ""
logLevel: "INFO"
serviceAccountTail: ".iam.gserviceaccount.com"
# FIXME GONRG-4910
partitionNamespace: ""
springProfilesActive: "gcp"
# bootstrap common
partition_name: "partition"
data_partition_id: ""
datafier_sa: "datafier"
partitionName: "partition"
dataPartitionId: ""
datafierSa: "datafier"
# bootstrap variables onprem
domain: ""
istioNamespace: "istio-ingress"
environment: ""
client_id: ""
clientId: "datafier"
conf:
configmap: "partition-config"
app_name: "partition"
# bootstrap common
bootstrap_name: "partition-bootstrap"
bootstrap_configmap: "partition-bootstrap-configmap"
on_prem_enabled: false
appName: "partition"
onPremEnabled: false
{{- if .Values.conf.on_prem_enabled }}
{{- if .Values.conf.onPremEnabled }}
{{- range $key, $spec := .Values.authorizations }}
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
......
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Values.conf.bootstrap_name }}"
name: "{{ .Values.conf.appName }}-bootstrap"
namespace: "{{ .Release.Namespace }}"
spec:
replicas: 1
selector:
matchLabels:
app: "{{ .Values.conf.bootstrap_name }}"
app: "{{ .Values.conf.appName }}-bootstrap"
template:
metadata:
labels:
app: "{{ .Values.conf.bootstrap_name }}"
app: "{{ .Values.conf.appName }}-bootstrap"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
containers:
- image: "{{ .Values.data.bootstrap_image }}"
name: "{{ .Values.conf.bootstrap_name }}"
- image: "{{ .Values.data.bootstrapImage }}"
name: "{{ .Values.conf.appName }}-bootstrap"
readinessProbe:
exec:
command:
......@@ -25,10 +25,10 @@ spec:
- /tmp/bootstrap_ready
envFrom:
- configMapRef:
name: "{{ .Values.conf.bootstrap_configmap }}"
name: "{{ .Values.conf.configmap }}-bootstrap"
resources:
limits:
cpu: "{{ .Values.data.bootstrap_limits_cpu }}"
memory: "{{ .Values.data.bootstrap_limits_memory }}"
cpu: "{{ .Values.data.bootstrapLimitsCpu }}"
memory: "{{ .Values.data.bootstrapLimitsMemory }}"
restartPolicy: Always
serviceAccountName: "{{ .Values.data.bootstrapServiceAccountName }}"
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Values.conf.app_name }}"
name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
replicas: 1
selector:
matchLabels:
app: "{{ .Values.conf.app_name }}"
app: "{{ .Values.conf.appName }}"
template:
metadata:
labels:
app: "{{ .Values.conf.app_name }}"
app: "{{ .Values.conf.appName }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
containers:
- image: "{{ .Values.data.image }}"
imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
name: "{{ .Values.conf.app_name }}"
name: "{{ .Values.conf.appName }}"
env:
- name: "ACCEPT_HTTP"
value: "true"
envFrom:
- configMapRef:
name: "{{ .Values.conf.configmap }}"
{{- if .Values.conf.on_prem_enabled }}
{{- if .Values.conf.onPremEnabled }}
- secretRef:
name: "{{ .Values.conf.secret }}"
{{- end }}
......@@ -36,9 +36,9 @@ spec:
- containerPort: 8080
resources:
requests:
cpu: "{{ .Values.data.requests_cpu }}"
memory: "{{ .Values.data.requests_memory }}"
cpu: "{{ .Values.data.requestsCpu }}"
memory: "{{ .Values.data.requestsMemory }}"
limits:
cpu: "{{ .Values.data.limits_cpu }}"
memory: "{{ .Values.data.limits_memory }}"
cpu: "{{ .Values.data.limitsCpu }}"
memory: "{{ .Values.data.limitsMemory }}"
serviceAccountName: "{{ .Values.data.serviceAccountName }}"
{{- if .Values.conf.on_prem_enabled }}
{{- if .Values.conf.onPremEnabled }}
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
......
apiVersion: v1
kind: Service
metadata:
name: "{{ .Values.conf.app_name }}"
name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
annotations:
cloud.google.com/neg: '{"ingress": true}'
......@@ -12,4 +12,4 @@ spec:
protocol: TCP
name: http
selector:
app: "{{ .Values.conf.app_name }}"
app: "{{ .Values.conf.appName }}"
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: "{{ .Values.conf.app_name }}"
name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
hosts:
{{- if .Values.conf.domain }}
- {{ printf "osdu.%s" .Values.conf.domain | quote }}
{{- else }}
- "*"
{{- end }}
gateways:
- service-gateway
http:
......@@ -16,4 +20,4 @@ spec:
- destination:
port:
number: 80
host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local"
host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local"
{{- if .Values.conf.on_prem_enabled }}
{{- if .Values.conf.onPremEnabled }}
apiVersion: v1
kind: ServiceAccount
metadata:
......
data:
requests_cpu: "0.1"
requests_memory: "260M"
limits_cpu: "1"
limits_memory: "1G"
requestsCpu: "0.1"
requestsMemory: "260M"
limitsCpu: "1"
limitsMemory: "1G"
serviceAccountName: "partition"
imagePullPolicy: "IfNotPresent"
image: ""
# bootstrap common
bootstrap_limits_cpu: "0.1"
bootstrap_limits_memory: "200M"
bootstrap_image: ""
# bootstrap
bootstrapLimitsCpu: "0.1"
bootstrapLimitsMemory: "200M"
bootstrapImage: ""
bootstrapServiceAccountName: ""
conf:
configmap: "partition-config"
secret: "partition-postgres-secret"
app_name: "partition"
# bootstrap common
bootstrap_name: "partition-bootstrap"
bootstrap_configmap: "partition-bootstrap-configmap"
on_prem_enabled: false
appName: "partition"
onPremEnabled: false
domain: ""
namespacePolicy:
mtlsMode: STRICT
authorizations:
......
......@@ -4,42 +4,41 @@ set -ex
DATA_PARTITION_ID_UPPER=${DATA_PARTITION_ID^^}
generate_post_data()
{
generate_post_data() {
cat <<EOF
{
"properties": {
"projectId": {
"sensitive": false,
"value": "${PROJECT_ID}"
"sensitive": false,
"value": "${PROJECT_ID}"
},
"serviceAccount": {
"sensitive": false,
"value": "${SERVICEACCOUNT}"
"sensitive": false,
"value": "${SERVICEACCOUNT}"
},
"complianceRuleSet": {
"sensitive": false,
"value": "shared"
"sensitive": false,
"value": "shared"
},
"dataPartitionId": {
"sensitive": false,
"value": "${DATA_PARTITION_ID}"
"sensitive": false,
"value": "${DATA_PARTITION_ID}"
},
"name": {
"sensitive": false,
"value": "${DATA_PARTITION_ID}"
"sensitive": false,
"value": "${DATA_PARTITION_ID}"
},
"policy-service-enabled": {
"sensitive": false,
"value": "false"
"sensitive": false,
"value": "false"
},
"bucket": {
"sensitive": false,
"value": "${PROJECT_ID}-records"
"sensitive": false,
"value": "${PROJECT_ID}-records"
},
"crmAccountID": {
"sensitive": false,
"value": "[${DATA_PARTITION_ID},${DATA_PARTITION_ID}]"
"sensitive": false,
"value": "[${DATA_PARTITION_ID},${DATA_PARTITION_ID}]"
},
"osm.postgres.datasource.url": {
"sensitive": true,
......@@ -89,12 +88,11 @@ generate_post_data()
"sensitive": true,
"value": "RABBITMQ_ADMIN_PASSWORD"
},
"oqm.rabbitmq.admin.schema": {
"oqm.rabbitmq.admin.schema": {
"sensitive": false,
"value": "http"
},
"oqm.rabbitmq.admin.host": {
"oqm.rabbitmq.admin.host": {
"sensitive": false,
"value": "rabbitmq"
},
......@@ -135,9 +133,6 @@ generate_post_data()
EOF
}
echo "sleep to prevent 500 response from the partition service, due to timeout of creation for Workload Identity"
sleep 20
if [ "$ENVIRONMENT" == "anthos" ]
then
......@@ -150,21 +145,25 @@ then
if [ "$status_code" == 201 ]
then
echo "partition bootstrap was OK!"
echo "Partition bootstrap finished successfully!"
elif [ "$status_code" == 409 ]
then
curl -X PATCH \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json" \
--data-raw "$(generate_post_data)"
echo "partition was patched because datastore has already had some entities!"
echo "Partition was patched because Postgres Database had already had entities!"
else
exit 1
fi
# FIXME "$ENVIRONMENT" == "gcp" or use another variable
elif [ "$ENVIRONMENT" == "" ]
then
echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity"
sleep 20
IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
SERVICEACCOUNT=${DATAFIER_SA}@${PROJECT_ID}.iam.gserviceaccount.com
......@@ -177,7 +176,7 @@ then
if [ "$status_code" == 201 ]
then
echo "partition bootstrap was OK!"
echo "Partition bootstrap finished successfully!"
elif [ "$status_code" == 409 ]
then
curl -X PATCH \
......@@ -185,7 +184,7 @@ then
-H "Authorization: Bearer ${IDENTITY_TOKEN}" \
-H "Content-Type: application/json" \
--data-raw "$(generate_post_data)"
echo "partition was patched because datastore has already had some entities!"
echo "Partition was patched because Datastore had already had entities!"
else
exit 1
fi
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment