Merge branch 'master' into airflow-add-variable

32bdc7e4 · harshit aggarwal · eeba6d75 · 4f196239 · 32bdc7e4 · 32bdc7e4
Commit 32bdc7e4 authored Sep 24, 2021 by harshit aggarwal
--- a/charts/airflow/helm-config-dp.yaml
+++ b/charts/airflow/helm-config-dp.yaml
@@ -331,8 +331,9 @@ airflow:
        "pyyaml==5.4.1",
        "requests==2.25.1",
        "tenacity==8.0.1",
-        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
-        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev31+59e58330.tar.gz"
+        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz",
+        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
+
    ]
    extraVolumeMounts:
        - name: azure-keyvault

--- a/charts/airflow/helm-config.yaml
+++ b/charts/airflow/helm-config.yaml
@@ -346,8 +346,8 @@ airflow:
        "pyyaml==5.4.1",
        "requests==2.25.1",
        "tenacity==8.0.1",
-        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
-        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev31+59e58330.tar.gz"
+        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz",
+        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
    ]
    extraVolumeMounts:
        - name: azure-keyvault

--- a/charts/osdu-common/templates/appgw-ingress.yaml
+++ b/charts/osdu-common/templates/appgw-ingress.yaml
@@ -121,3 +121,7 @@ spec:
              serviceName: dataset
              servicePort: 80
            path: /api/dataset/v1/*
+          - backend:
+              serviceName: seismic-dms-file-metadata-service
+              servicePort: 80
+            path: /seismic-file-metadata/api/v1/*   
--- a/charts/osdu-istio/templates/cert-checker-cronjob.yaml
+++ b/charts/osdu-istio/templates/cert-checker-cronjob.yaml
@@ -51,6 +51,11 @@ spec:
                configMapKeyRef:
                  name: {{ .Values.global.job.configmap_name }}
                  key: ENV_CLUSTER_NAME
+            - name: ENV_APPGW_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ .Values.global.job.configmap_name }}
+                  key: ENV_APPGW_NAME
            command:
            - /bin/sh
            args:
@@ -62,6 +67,7 @@ spec:

              # Compare expire dates of certificates in Key Vault and in istio-system namespaces
              function check_expire_date() {
+                echo "Compare expire dates of certificates in Key Vault and in istio-system namespaces"
                az keyvault certificate download  --vault-name  ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} --file ${KV_CERT_NAME}.pem
                KV_CERT_EXPIREDATE=$(openssl x509 -in ${KV_CERT_NAME}.pem -enddate -noout |  cut -d '=' -f2)
                KV_CERT_EXPIREDATE=$(date "+%Y-%m-%d" --date="${KV_CERT_EXPIREDATE}")
@@ -84,7 +90,7 @@ spec:

              # Cleanup function
              cleanup() {
-                echo Clean all existing files
+                echo "Clean all existing files"
                rm -f cert.crt cert.key osdu-certificate.pfx ${KV_CERT_NAME}.pem
                curl -X POST "http://localhost:${SIDECAR_PORT}/quitquitquit"
              }
@@ -111,7 +117,7 @@ spec:
              # Log In in Azure
              az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}

-{{- if .Values.global.istio.enableIstioKeyvaultCert  }}
+{{- if .Values.global.istio.enableIstioKeyvaultCert }}

              K8S_CERT_SECRET=istio-appgw-ssl-cert
              K8S_NAMESPACE_NAME=istio-system
@@ -120,8 +126,13 @@ spec:
              check_expire_date

              # Download BYOC certificate from keyvault
+              echo "Download BYOC certificate from keyvault"
              az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}

+              # Upload BYOC certificate to appgw
+              echo "Upload BYOC certificate to appgw" 
+              az network application-gateway ssl-cert update -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
+
              # Extract key and crt
              openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
              openssl rsa -in cert.pem -out cert.key
@@ -135,6 +146,9 @@ spec:

              sleep 5

+              # Restart istio-ingressgateway pods
+              kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
+
              # Check certificate expire date one more time
              check_expire_date


--- a/charts/osdu-istio/templates/cert-init-job.yaml
+++ b/charts/osdu-istio/templates/cert-init-job.yaml
@@ -43,6 +43,11 @@ spec:
            configMapKeyRef:
              name: {{ .Values.global.job.configmap_name }}
              key: ENV_CLUSTER_NAME
+        - name: ENV_APPGW_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Values.global.job.configmap_name }}
+              key: ENV_APPGW_NAME        
        command:
        - /bin/sh
        args:
@@ -82,14 +87,19 @@ spec:
          az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
          az aks get-credentials --resource-group ${ENV_SR_GROUP_NAME} --name ${ENV_CLUSTER_NAME}

-{{- if .Values.global.istio.enableIstioKeyvaultCert  }}
+{{- if .Values.global.istio.enableIstioKeyvaultCert }}

          K8S_CERT_SECRET=istio-appgw-ssl-cert
          K8S_NAMESPACE_NAME=istio-system

          # Download BYOC certificate from keyvault
+          echo "Download BYOC certificate from keyvault"
          az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}

+          # Upload BYOC certificate to appgw
+          echo "Upload BYOC certificate to appgw" 
+          az network application-gateway ssl-cert create -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
+          
          # Extract key and crt
          openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
          openssl rsa -in cert.pem -out cert.key
@@ -101,6 +111,9 @@ spec:
          --from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
          -o yaml | kubectl apply -f -

+          # Restart istio-ingressgateway pods
+          kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
+
 {{ else }}
          
          K8S_CERT_SECRET=osdu-certificate

--- a/infra/templates/osdu-r3-mvp/service_resources/config_map.tf
+++ b/infra/templates/osdu-r3-mvp/service_resources/config_map.tf
@@ -66,6 +66,7 @@ resource "kubernetes_config_map" "appgw_configmap" {
    ENV_SR_GROUP_NAME = azurerm_resource_group.main.name
    ENV_KEYVAULT_NAME = data.terraform_remote_state.central_resources.outputs.keyvault_name
    ENV_CLUSTER_NAME  = module.aks.name
+    ENV_APPGW_NAME    = module.istio_appgateway[count.index].name
  }
-  depends_on = [kubernetes_namespace.osdu]
+  depends_on = [kubernetes_namespace.osdu, module.istio_appgateway]
 }
--- a/infra/templates/osdu-r3-mvp/service_resources/main.tf
+++ b/infra/templates/osdu-r3-mvp/service_resources/main.tf
@@ -441,6 +441,39 @@ data "azurerm_resource_group" "aks_node_resource_group" {
  name = module.aks.node_resource_group
 }

+// Give AD Principal Access rights to Change the Istio Application Gateway
+resource "azurerm_role_assignment" "agic_istio_appgw_contributor" {
+  count = var.feature_flag.autoscaling ? 1 : 0
+
+  principal_id         = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
+  scope                = module.istio_appgateway[count.index].id
+  role_definition_name = "Contributor"
+
+  depends_on = [module.istio_appgateway]
+}
+
+// Give AD Principal Access rights to Operate the Istio Application Gateway Identity
+resource "azurerm_role_assignment" "agic_istio_app_gw_contributor_for_adsp" {
+  count = var.feature_flag.autoscaling ? 1 : 0
+
+  principal_id         = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
+  scope                = module.istio_appgateway[count.index].managed_identity_resource_id
+  role_definition_name = "Managed Identity Operator"
+
+  depends_on = [module.istio_appgateway]
+}
+
+// Give AD Principal the rights to look at the Resource Group
+resource "azurerm_role_assignment" "agic_istio_resourcegroup_reader" {
+  count = var.feature_flag.autoscaling ? 1 : 0
+
+  principal_id         = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
+  scope                = azurerm_resource_group.main.id
+  role_definition_name = "Reader"
+
+  depends_on = [module.istio_appgateway]
+}
+
 // Give AKS Access rights to Operate the Node Resource Group
 resource "azurerm_role_assignment" "all_mi_operator" {
  principal_id         = module.aks.kubelet_object_id

--- a/infra/templates/osdu-r3-mvp/service_resources/tests/unit/unit_test.go
+++ b/infra/templates/osdu-r3-mvp/service_resources/tests/unit/unit_test.go
@@ -36,8 +36,8 @@ var tfOptions = &terraform.Options{
 }

 var istioEnabled = os.Getenv("AUTOSCALING_ENABLED")
-var istioResourses = 11
-var totalResources = 138
+var istioResourses = 14
+var totalResources = 141

 func TestTemplate(t *testing.T) {
 	expectedAppDevResourceGroup := asMap(t, `{

--- a/tools/helper_services_data_seeding_agent/Dockerfile
+++ b/tools/helper_services_data_seeding_agent/Dockerfile
+FROM mcr.microsoft.com/azure-cli
+
+RUN apk update
+RUN apk add ca-certificates wget && update-ca-certificates
+RUN apk add  libc6-compat
+RUN apk add git
+
+RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
+    && wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
+    && chmod +x /usr/local/bin/kubectl \
+    && chmod g+rwx /root \
+    && mkdir /config \
+    && chmod g+rwx /config \
+    && apk add --update coreutils && rm -rf /var/cache/apk/*
+
+WORKDIR /usr/src/app
+
+COPY transfer.sh ./
+
+RUN chmod +x transfer.sh
+ENTRYPOINT [ "./transfer.sh" ]
\ No newline at end of file
--- a/tools/helper_services_data_seeding_agent/docker-compose.yaml
+++ b/tools/helper_services_data_seeding_agent/docker-compose.yaml
+version: '3'
+services:
+  helper-services-data-seeding-agent:
+    build:
+      context: ../helper_services_data_seeding_agent
+      dockerfile: ./Dockerfile
+    image: msosdu.azurecr.io/helper-services-data-seeding-agent:$VERSION
+    environment:
+      OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
+      RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
+      CONFIG_MAP_NAME: $CONFIG_MAP_NAME
+      VERSION: $VERSION
\ No newline at end of file
--- a/tools/helper_services_data_seeding_agent/transfer.sh
+++ b/tools/helper_services_data_seeding_agent/transfer.sh
+#!/bin/bash
+
+mkdir -p tmp
+cd tmp
+wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
+cp ./azcopy /usr/bin/
+cd ..
+currentStatus=""
+currentMessage=""
+
+git clone https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service.git
+if [[ $? -gt 0 ]]; then
+  currentMessage="failure"
+  currentMessage="${currentMessage}. Failed to clone crs-conversion-service"
+fi
+git clone https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service.git
+if [[ $? -gt 0 ]]; then
+  currentMessage="failure"
+  currentMessage="${currentMessage}. Failed to clone crs-catalog-service"
+fi
+git clone https://community.opengroup.org/osdu/platform/system/reference/unit-service.git
+if [[ $? -gt 0 ]]; then
+  currentMessage="failure"
+  currentMessage="${currentMessage}. Failed to clone unit-service"
+fi
+CRS_CONVERSION_SOURCE_FOLDER="crs-conversion-service/apachesis_setup"
+CRS_CATALOG_SOURCE_FOLDER="crs-catalog-service/data/crs_catalog_v2.json"
+UNIT_SOURCE_FOLDER="unit-service/data/unit_catalog_v2.json"
+
+az login --identity --username $OSDU_IDENTITY_ID
+ENV_VAULT=$(az keyvault list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
+STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv)
+if [ -z "$STORAGE_ACCOUNT_NAME" -a "$STORAGE_ACCOUNT_NAME" == " " ]; then
+  currentStatus="failure"
+  currentMessage="${currentMessage}. Storage Account Name Not Found. "
+fi
+STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv)
+if [ -z "$STORAGE_ACCOUNT_KEY" -a "$STORAGE_ACCOUNT_KEY" == " " ]; then
+  currentStatus="failure"
+  currentMessage="${currentMessage}. Storage Account Key Not Found. "
+else
+
+  EXPIRE=$(date -u -d "59 minutes" '+%Y-%m-%dT%H:%M:%SZ')
+  START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
+
+  #Generating the SAS Token required for Authorization
+  AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $STORAGE_ACCOUNT_KEY --start $START --expiry $EXPIRE --https-only --resource-types sco --services f --permissions cwdlur -o tsv)
+  azcopy cp $CRS_CONVERSION_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/crs-conversion?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
+  if [[ $? -gt 0 ]]; then
+    currentStatus="failure"
+    currentMessage="${currentMessage}. Failed to copy data to crs-conversion file share"
+  fi
+  azcopy cp $CRS_CATALOG_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/crs?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
+  if [[ $? -gt 0 ]]; then
+    currentStatus="failure"
+    currentMessage="${currentMessage}. Failed to copy data to crs file share"
+  fi
+  azcopy cp $UNIT_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/unit?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
+  if [[ $? -gt 0 ]]; then
+    currentStatus="failure"
+    currentMessage="${currentMessage}. Failed to copy data to unit file share"
+  fi
+fi
+
+if [ -z "$currentStatus" -a "$currentStatus"==" " ]; then
+  currentStatus="success"
+fi
+echo "Current Status: ${currentStatus}"
+echo "Current Message: ${currentMessage}"
+
+if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
+  az login --identity --username $OSDU_IDENTITY_ID
+  ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
+  az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
+  kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
+
+  Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
+  Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
+
+  Message="${Message}Helper Data Seeding Message: ${currentMessage}. "
+
+  ## Update ConfigMap
+  kubectl create configmap $CONFIG_MAP_NAME \
+    --from-literal=status="$currentStatus" \
+    --from-literal=message="$Message" \
+    -o yaml --dry-run=client | kubectl replace -f -
+fi
+
+if [[ ${currentStatus} == "success" ]]; then
+  exit 0
+else
+  exit 1
+fi