Commit d0820524 authored by dzmitry_paulouski's avatar dzmitry_paulouski
Browse files

Add feature flag, zones support, reconfigure istio and add jobs

parent d40ca0d6
Pipeline #49991 passed with stages
in 59 seconds
......@@ -37,7 +37,7 @@ global:
istio:
username: #{istio-username}#
password: #{istio-password}#
loadBalancerIP: #{istio_int_load_balancer_ip}#
loadBalancerIP: #{istio-int-load-balancer-ip}#
dns_host: #{DNS_HOST}#
################################################################################
# Specify the resource limits
......@@ -47,3 +47,8 @@ global:
defaultMemoryRequests: "4Gi"
defaultCpuLimits: "1"
defaultMemoryLimits: "4Gi"
job:
name: cert-checker
image: mcr.microsoft.com/azure-cli
configmap_name: osdu-istio-appgw-cert
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: {{ .Values.global.job.name }}
namespace: osdu
labels:
app: "{{ .Values.global.job.name }}"
spec:
schedule: "0 * * * *"
jobTemplate:
metadata:
name: "{{ .Values.global.job.name }}"
labels:
app: "{{ .Values.global.job.name }}"
type: "job"
spec:
template:
spec:
containers:
- name: cert-checker
image: {{ .Values.global.job.image }}
env:
- name: client_id
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: client_secret
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: tenant_id
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: ENV_SR_GROUP_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_SR_GROUP_NAME
- name: ENV_KEYVAULT_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_KEYVAULT_NAME
- name: ENV_CLUSTER_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_CLUSTER_NAME
command:
- /bin/sh
args:
- -c
- |
SIDECAR_PORT=15020
KV_CERT_NAME=appgw-ssl-cert
K8S_ISTIONAMESPACE_NAME=istio-system
# Compare expire dates of certificates in Key Vault and in istio-system namespaces
function check_expire_date() {
az keyvault certificate download --vault-name ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} --file ${KV_CERT_NAME}.pem
KV_CERT_EXPIREDATE=$(openssl x509 -in ${KV_CERT_NAME}.pem -enddate -noout | cut -d '=' -f2)
KV_CERT_EXPIREDATE=$(date "+%Y-%m-%d" --date="${KV_CERT_EXPIREDATE}")
az aks get-credentials --resource-group ${ENV_SR_GROUP_NAME} --name ${ENV_CLUSTER_NAME}
kubectl get secret -n ${K8S_ISTIONAMESPACE_NAME} ${K8S_CERT_SECRET} -o jsonpath="{ .data.tls\.crt }" | base64 -d > cert.crt
SECRETEXPIREDATE=$(openssl x509 -in cert.crt -enddate -noout | cut -d '=' -f2)
SECRETEXPIREDATE=$(date "+%Y-%m-%d" --date="${SECRETEXPIREDATE}")
if [ ${KV_CERT_EXPIREDATE} = ${SECRETEXPIREDATE} ]; then
echo "KV_CERT_EXPIREDATE: ${KV_CERT_EXPIREDATE} and SECRETEXPIREDATE: ${SECRETEXPIREDATE}"
echo "The cert in secret is up to date"
exit 0
else
echo "KV_CERT_EXPIREDATE: ${KV_CERT_EXPIREDATE} and SECRETEXPIREDATE: ${SECRETEXPIREDATE}"
echo "The cert in secret is not up to date"
fi
rm -f ${KV_CERT_NAME}.pem
}
# Cleanup function
cleanup() {
echo Clean all existing files
rm -f cert.crt cert.key osdu-certificate.pfx ${KV_CERT_NAME}.pem
curl -X POST "http://localhost:${SIDECAR_PORT}/quitquitquit"
}
trap cleanup 0 2 3 6 ERR
set -e
# Wait for internet connection
until nc -z google.com 80
do
sleep 1
done
# Install kubectl
if [ ! -x /usr/local/bin/kubectl ]; then
echo "Download and install kubectl..."
curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod a+x /usr/local/bin/kubectl
fi
# Install coreutils
apk add --update coreutils
# Log In in Azure
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
{{- if .Values.global.ingress.enableKeyvaultCert }}
K8S_CERT_SECRET=appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
# Check certificate expire date.
check_expire_date
# Download BYOC certificate from keyvault
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
openssl crl2pkcs7 -nocrl -certfile cert.pem | openssl pkcs7 -print_certs -out cert.crt
# Create secret for istio-ingressgateway in istio-system namespace
kubectl create secret -n ${K8S_ISTIONAMESPACE_NAME} generic ${K8S_CERT_SECRET} \
--save-config --dry-run=client \
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
sleep 5
# Check certificate expire date one more time
check_expire_date
echo "Cannot update cert in secret"
exit 1
{{ else }}
K8S_CERT_SECRET=osdu-certificate
K8S_NAMESPACE_NAME=osdu
# Check certificate expire date.
check_expire_date
# Get osdu-certificate secret created by cert-manager in osdu namespace and extract key and crt
kubectl get secret -n ${K8S_NAMESPACE_NAME} ${K8S_CERT_SECRET} -o jsonpath="{ .data.tls\.crt }" | base64 -d > cert.crt
kubectl get secret -n ${K8S_NAMESPACE_NAME} ${K8S_CERT_SECRET} -o jsonpath="{ .data.tls\.key }" | base64 -d > cert.key
# Create secret for istio-ingressgateway in istio-system namespace from key and crt
kubectl create secret -n ${K8S_ISTIONAMESPACE_NAME} generic ${K8S_CERT_SECRET} \
--save-config --dry-run=client \
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
# Create certificate file
openssl pkcs12 \
-passout pass: \
-export \
-out osdu-certificate.pfx \
-in cert.crt \
-inkey cert.key
# Upload certificate file in Key Vault to use in in AppGW
az keyvault certificate import --vault-name ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} -f osdu-certificate.pfx
sleep 5
# Check certificate expire date one more time
check_expire_date
echo "Cannot update KV cert"
exit 1
{{- end }}
restartPolicy: Never
apiVersion: batch/v1
kind: Job
metadata:
name: {{ .Values.global.job.name }}
namespace: osdu
labels:
app: "{{ .Values.global.job.name }}"
spec:
template:
spec:
containers:
- name: cert-checker
image: {{ .Values.global.job.image }}
env:
- name: client_id
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: client_secret
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: tenant_id
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: ENV_SR_GROUP_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_SR_GROUP_NAME
- name: ENV_KEYVAULT_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_KEYVAULT_NAME
- name: ENV_CLUSTER_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_CLUSTER_NAME
command:
- /bin/sh
args:
- -c
- |
SIDECAR_PORT=15020
KV_CERT_NAME=appgw-ssl-cert
K8S_ISTIONAMESPACE_NAME=istio-system
# Cleanup function
cleanup() {
echo Clean all existing files
rm -f cert.crt cert.key osdu-certificate.pfx ${KV_CERT_NAME}.pem
curl -X POST "http://localhost:${SIDECAR_PORT}/quitquitquit"
}
trap cleanup 0 2 3 6 ERR
set -e
# Wait for internet connection
until nc -z google.com 80
do
sleep 1
done
# Install kubectl
if [ ! -x /usr/local/bin/kubectl ]; then
echo "Download and install kubectl..."
curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod a+x /usr/local/bin/kubectl
fi
# Install coreutils
apk add --update coreutils
# Log in in Azure and get cluster credentials
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
az aks get-credentials --resource-group ${ENV_SR_GROUP_NAME} --name ${ENV_CLUSTER_NAME}
{{- if .Values.global.ingress.enableKeyvaultCert }}
K8S_CERT_SECRET=appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
# Download BYOC certificate from keyvault
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
openssl crl2pkcs7 -nocrl -certfile cert.pem | openssl pkcs7 -print_certs -out cert.crt
# Create secret for istio-ingressgateway in istio-system namespace
kubectl create secret -n ${K8S_ISTIONAMESPACE_NAME} generic ${K8S_CERT_SECRET} \
--save-config --dry-run=client \
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
{{ else }}
K8S_CERT_SECRET=osdu-certificate
K8S_NAMESPACE_NAME=osdu
# Get osdu-certificate secret created by cert-manager in osdu namespace and extract key and crt
kubectl get secret -n ${K8S_NAMESPACE_NAME} ${K8S_CERT_SECRET} -o jsonpath="{ .data.tls\.crt }" | base64 -d > cert.crt
kubectl get secret -n ${K8S_NAMESPACE_NAME} ${K8S_CERT_SECRET} -o jsonpath="{ .data.tls\.key }" | base64 -d > cert.key
# Create secret for istio-ingressgateway in istio-system namespace from key and crt
kubectl create secret -n ${K8S_ISTIONAMESPACE_NAME} generic ${K8S_CERT_SECRET} \
--save-config --dry-run=client \
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
# Create certificate file
openssl pkcs12 \
-passout pass: \
-export \
-out osdu-certificate.pfx \
-in cert.crt \
-inkey cert.key
# Upload certificate file in Key Vault to use in in AppGW
az keyvault certificate import --vault-name ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} -f osdu-certificate.pfx
{{- end }}
restartPolicy: Never
......@@ -15,4 +15,8 @@ spec:
- "{{ .Values.global.istio.dns_host }}"
tls:
mode: SIMPLE
{{- if .Values.global.ingress.enableKeyvaultCert }}
credentialName: appgw-ssl-cert
{{ else }}
credentialName: osdu-certificate
{{- end }}
......@@ -10,6 +10,14 @@ spec:
- istio-gateway
http:
- match:
- uri:
exact: "/"
route:
- destination:
host: osdu-common
port:
number: 80
- match:
- uri:
prefix: "/api/partition/v1"
route:
......
......@@ -84,6 +84,8 @@ jobs:
- template: tasks/tests-unit.yml
parameters:
skip: ${{ parameters.skipTests }}
terraformWorkspacePrefix: ${{ parameters.configuration.terraformWorkspacePrefix }}
- template: tasks/tf-plan.yml
parameters:
......
......@@ -14,11 +14,14 @@
parameters:
skip: false
terraformWorkspacePrefix: ''
steps:
- task: AzureCLI@1
displayName: 'Run Unit Tests'
env:
TERRAFORM_WORKSPACE_PREFIX: ${{ parameters.terraformWorkspacePrefix }}
TF_VAR_remote_state_container: $(TF_VAR_remote_state_container)
TF_VAR_remote_state_account: $(TF_VAR_remote_state_account)
TF_VAR_resource_group_location: $(TF_VAR_resource_group_location)
......@@ -26,7 +29,6 @@ steps:
TF_VAR_elasticsearch_username: $(elastic-username-${{ parameters.terraformWorkspacePrefix }}-${{ parameters.environmentName }})
TF_VAR_elasticsearch_password: $(elastic-password-${{ parameters.terraformWorkspacePrefix }}-${{ parameters.environmentName }})
TF_VAR_elasticsearch_endpoint: $(elastic-endpoint-${{ parameters.terraformWorkspacePrefix }}-${{ parameters.environmentName }})
TF_VAR_common_resources_workspace_name: $(TF_VAR_common_resources_workspace_name)
TF_VAR_central_resources_workspace_name: $(TF_VAR_central_resources_workspace_name)
TF_VAR_data_resources_workspace_name: $(TF_VAR_data_resources_workspace_name)
TF_VAR_data_partition_name: $(TF_VAR_data_partition_name)
......@@ -38,6 +40,15 @@ steps:
TF_VAR_principal_password: $(TF_VAR_principal_password)
TF_VAR_principal_objectId: $(TF_VAR_principal_objectId)
TF_VAR_gitops_path: $(TF_VAR_gitops_path)
TF_VAR_aks_services_agent_vm_count: $(TF_VAR_aks_services_agent_vm_count)
TF_VAR_aks_services_agent_vm_maxcount: $(TF_VAR_aks_services_agent_vm_maxcount)
TF_VAR_aks_services_agent_vm_size: $(TF_VAR_aks_services_agent_vm_size)
TF_VAR_aks_system_agent_vm_count: $(TF_VAR_aks_system_agent_vm_count)
TF_VAR_aks_system_agent_vm_maxcount: $(TF_VAR_aks_system_agent_vm_maxcount)
TF_VAR_aks_system_agent_vm_size: $(TF_VAR_aks_system_agent_vm_size)
TF_VAR_istio_int_load_balancer_ip: $(TF_VAR_istio_int_load_balancer_ip)
TF_VAR_aks_dns_host: $(TF_VAR_aks_dns_host)
condition: not(coalesce(variables.SKIP_TESTS, ${{ parameters.skip }}))
inputs:
......@@ -54,6 +65,16 @@ steps:
export ARM_ACCESS_KEY=$(az storage account keys list --subscription "$ARM_SUBSCRIPTION_ID" --account-name "$(TF_VAR_remote_state_account)" --query "[0].value" --output tsv)
cd "$ARTIFACT_ROOT"/"$TERRAFORM_TEMPLATE_PATH"
if [[ "$TERRAFORM_WORKSPACE_PREFIX" == sr* ]]; then
# Check if istio appgw enable
pushd $(TF_TEMPLATE_WORKING_DIR)
[ ! -f $TF_VARFILE ] && TF_VARFILE=override.tfvars
AUTOSCALING_ENABLED=$(echo 'var.feature_flag["autoscaling"]' | terraform console -var-file=$TF_VARFILE)
echo "AUTOSCALING_ENABLED : ${AUTOSCALING_ENABLED}"
export AUTOSCALING_ENABLED
popd
fi
# Setting the scripts to be run as executable
chmod -fR 755 *.sh || true
......
......@@ -178,6 +178,7 @@ resource "azurerm_application_gateway" "main" {
port = 443
protocol = "Https"
request_timeout = 1
host_name = length(var.host_name) == 0 ? null : var.host_name
}
backend_address_pool {
......@@ -192,6 +193,8 @@ resource "azurerm_application_gateway" "main" {
min_protocol_version = var.ssl_policy_min_protocol_version
}
zones = var.gateway_zones
lifecycle {
ignore_changes = [
ssl_certificate,
......
......@@ -114,3 +114,15 @@ variable "backend_address_pool_fqdns" {
type = list(any)
default = []
}
variable "host_name" {
description = "A DNS name whis will use for APPGW backend http setting"
type = string
default = ""
}
variable "gateway_zones" {
description = ""
type = list(string)
default = null
}
......@@ -56,6 +56,8 @@ resource "kubernetes_config_map" "osduconfigmap" {
}
resource "kubernetes_config_map" "appgw_configmap" {
count = var.feature_flag.autoscaling ? 1 : 0
metadata {
name = "osdu-istio-appgw-cert"
namespace = local.osdu_ns
......
......@@ -95,8 +95,10 @@ resource "azurerm_monitor_diagnostic_setting" "gw_diagnostics" {
}
resource "azurerm_monitor_diagnostic_setting" "istio_gw_diagnostics" {
count = var.feature_flag.autoscaling ? 1 : 0
name = "istio_gw_diagnostics"
target_resource_id = module.istio_appgateway.id
target_resource_id = module.istio_appgateway[count.index].id
log_analytics_workspace_id = data.terraform_remote_state.central_resources.outputs.log_analytics_id
......@@ -135,6 +137,8 @@ resource "azurerm_monitor_diagnostic_setting" "istio_gw_diagnostics" {
enabled = local.retention_policy
}
}
depends_on = [module.istio_appgateway]
}
#-------------------------------
......
......@@ -30,7 +30,7 @@ terraform {
required_providers {
azurerm = {
source = "hashicorp/azurerm"
version = "=2.41.0"
version = "=2.64.0"
}
azuread = {
source = "hashicorp/azuread"
......@@ -127,6 +127,19 @@ locals {
aks_identity_name = format("%s-pod-identity", local.aks_cluster_name)
aks_dns_prefix = local.base_name_60
availability_zones = [
"1",
"2",
"3"
]
gateway_zones = [
"1",
"2",
"3"
]
role = "Contributor"
rbac_principals = [
// OSDU Identity
......@@ -293,6 +306,8 @@ module "appgateway" {
}
module "istio_appgateway" {
count = var.feature_flag.autoscaling ? 1 : 0
source = "../../../modules/providers/azure/appgw"
name = local.istio_app_gw_name
......@@ -308,9 +323,12 @@ module "istio_appgateway" {
ssl_policy_min_protocol_version = var.ssl_policy_min_protocol_version
backend_address_pool_ips = var.istio_int_load_balancer_ip == "" ? null : [var.istio_int_load_balancer_ip]
gateway_zones = local.gateway_zones
resource_tags = var.resource_tags
min_capacity = var.appgw_min_capacity
max_capacity = var.appgw_max_capacity
host_name = var.aks_dns_host
}
// Give AGIC Identity Access rights to Change the Application Gateway
......@@ -320,13 +338,6 @@ resource "azurerm_role_assignment" "appgwcontributor" {
role_definition_name = "Contributor"
}
resource "azurerm_key_vault_access_policy" "kv_policy_for_adsp" {
key_vault_id = data.terraform_remote_state.central_resources.outputs.keyvault_id
tenant_id = azurerm_function_app.app_func.identity.0.tenant_id
object_id = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
certificate_permissions = ["get", "list", "import", "update"]
}
// Give AGIC Identity the rights to look at the Resource Group
resource "azurerm_role_assignment" "agic_resourcegroup_reader" {
principal_id = azurerm_user_assigned_identity.agicidentity.principal_id
......@@ -370,11 +381,14 @@ module "aks" {
}
resource "azurerm_kubernetes_cluster_node_pool" "services" {
count = var.feature_flag.autoscaling ? 1 : 0
name = "services"
kubernetes_cluster_id = module.aks.id
node_count = var.aks_services_agent_vm_count
min_count = var.aks_services_agent_vm_count
max_count = var.aks_services_agent_vm_maxcount
availability_zones = local.availability_zones
vnet_subnet_id = module.network.subnets.1
orchestrator_version = var.kubernetes_version
vm_size = var.aks_services_agent_vm_size
......@@ -406,6 +420,8 @@ resource "azurerm_role_assignment" "vm_contributor" {
// Give AKS Access to Operate the Network
resource "azurerm_role_assignment" "subnet_contributor" {
count = var.feature_flag.autoscaling ? 1 : 0
principal_id = module.aks.principal_id
scope = module.network.subnets.1
role_definition_name = "Contributor"
......@@ -441,170 +457,13 @@ resource "azurerm_role_assignment" "osdu_identity_mi_operator" {
# // Give AD Principal Access rights to AKS cluster
resource "azurerm_role_assignment" "aks_contributor" {
count = var.feature_flag.autoscaling ? 1 : 0
principal_id = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
scope = module.aks.id
role_definition_name = "Contributor"
}
// Cronjob for updating istio-appgw cert
resource "kubernetes_cron_job" "cert-checker" {
metadata {
name = "cert-checker"
namespace = "osdu"
}
spec {
concurrency_policy = "Replace"
failed_jobs_history_limit = 5
schedule = "0 * * * *"
starting_deadline_seconds = 10
successful_jobs_history_limit = 10
job_template {
metadata {}
spec {
backoff_limit = 2
ttl_seconds_after_finished = 10
template {
metadata {}
spec {
container {