diff --git a/.gitignore b/.gitignore index 9c52e32e7795c2781e023c05c4ce8f989384c05f..da93b46aa9128ef5613f179a7e338a026a1f9100 100644 --- a/.gitignore +++ b/.gitignore @@ -375,3 +375,4 @@ build/ # Dep files *.toml *.lock +.aider* diff --git a/README.md b/README.md index 05815d031d2f7c147dd326321896dcfe25e7b11e..f61930fd969a9d89503b5fe516cf0c12470c124d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Azure environment cost ballpark [estimate](https://azure.com/e/2a3cb4a935954d7ea ### Install the required tooling -This document assumes one is running a current version of Ubuntu. Windows users can install the Ubuntu Terminal from the Microsoft Store. The Ubuntu Terminal enables Linux command-line utilities, including bash, ssh, and git that will be useful for the following deployment. +This document assumes one is running a current version of Ubuntu. Windows users can install the Ubuntu Terminal from the Microsoft Store. The Ubuntu Terminal enables Linux command-line utilities, including bash, ssh, and git that will be useful for the following deployment. <b>_Note:</b> You will need the Windows Subsystem for Linux installed to use the Ubuntu Terminal on Windows_. @@ -41,11 +41,11 @@ az account set --subscription <your_subscription> ### [Optional] Create a Flux Manifest Repository - <b>Note:</b> This step is optional and not recommended one for new installations. Azure supports [Helm Charts](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure), instead of flux for all latest releases. + <b>Note:</b> This step is optional and not recommended one for new installations. Azure supports [Helm Charts](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure), instead of flux for all latest releases. > [Steps to create Flux Manifest Repository](./docs/flux.md) - - - + + + ### Provision the Common Resources > [Role Documentation](https://docs.microsoft.com/en-us/azure/role-based-access-control/rbac-and-directory-admin-roles): Provisioning Common Resources requires owner access to the subscription, however AD Service Principals are created that will required an AD Admin to grant approval consent on the principals created. @@ -101,7 +101,7 @@ az keyvault secret set --vault-name $COMMON_VAULT --name "istio-password" --valu ``` ### Elastic Search Setup -Infrastructure requires a bring your own Elastic Search Instance of a version of 7.x (ie: 7.11.1) with a valid https endpoint and the access information must now be stored in the Common KeyVault. The recommended method of Elastic Search is to create a AKS cluster and install ECK with [helm charts](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure/-/tree/master/osdu-eck-community) following instructions [here](./docs/elastic-helm.md). +Infrastructure requires a bring your own Elastic Search Instance of a version of 8.x (ie: 8.16.1) with a valid https endpoint and the access information must now be stored in the Common KeyVault. The recommended method of Elastic Search is to create a AKS cluster and install ECK following instructions [here](./docs/elastic-setup.md). ```bash diff --git a/docs/elastic-setup.md b/docs/elastic-setup.md new file mode 100644 index 0000000000000000000000000000000000000000..107f5bcae412e8b46de3141d7e4344a9e9ecad63 --- /dev/null +++ b/docs/elastic-setup.md @@ -0,0 +1,381 @@ +# Elastic Search Setup Guide + + +## Overview +This guide describes how to set up and manage Elasticsearch clusters using ECK (Elastic Cloud on Kubernetes). ECK provides a Kubernetes operator that simplifies the deployment, management, and orchestration of Elasticsearch clusters on Kubernetes. + +With ECK, you can: +- Deploy and manage multiple Elasticsearch clusters +- Automate cluster scaling and upgrades +- Handle secure cluster configuration +- Manage cluster monitoring and backups +- Integrate with Kibana for visualization + +This guide will walk you through the process of setting up ECK and deploying Elasticsearch clusters in a Kubernetes environment. + + +## Prerequisites + +### Create Azure Resources + +#### AKS Cluster with Node Auto Provisioning + +```bash +RESOURCE_GROUP=elastic-cluster +LOCATION=centralus +CLUSTER_NAME=elastic-aks + +# Create Resource Group +az group create --name $RESOURCE_GROUP --location $LOCATION + +# Create Cluster +az aks create --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME \ + --node-provisioning-mode Auto \ + --network-plugin azure \ + --network-plugin-mode overlay \ + --network-dataplane cilium \ + --generate-ssh-keys +``` + +#### Storage Account with Container + +```bash +# Set variables +STORAGE_ACCOUNT_NAME="<storage-account-name>" +CONTAINER_NAME="elasticsearch-snapshots" + +# Create storage account +az storage account create \ + --name "$STORAGE_ACCOUNT_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --location "$LOCATION" \ + --sku Standard_RAGRS \ + --kind StorageV2 \ + --min-tls-version TLS1_2 \ + --enable-hierarchical-namespace false \ + --allow-blob-public-access false + +# Create container for snapshots +az storage container create \ + --name "$CONTAINER_NAME" \ + --account-name "$STORAGE_ACCOUNT_NAME" +``` + +#### User Assigned Managed Identity + +```bash +# Set variables +USER_ASSIGNED_IDENTITY_NAME="es-snapshot-identity" + +# Create the managed identity +az identity create \ + --name "$USER_ASSIGNED_IDENTITY_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --location "$LOCATION" + +# Store the identity's client ID and resource ID +USER_ASSIGNED_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $USER_ASSIGNED_IDENTITY_NAME --query 'clientId' -o tsv) +USER_ASSIGNED_RESOURCE_ID=$(az identity show -g $RESOURCE_GROUP -n $USER_ASSIGNED_IDENTITY_NAME --query 'id' -o tsv) +``` + +#### Assign Storage Blob Data Contributor Role to the Managed Identity: + +```bash +az role assignment create \ + --role "Storage Blob Data Contributor" \ + --assignee "$USER_ASSIGNED_CLIENT_ID" \ + --scope "/subscriptions/$SUBSCRIPTION/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" +``` + +#### Create Federated Credential + +```bash +NAMESPACE=elastic-search + +# Retrieve AKS OIDC issuer +AKS_OIDC_ISSUER=$(az aks show --name "$CLUSTER_NAME" --resource-group "$RESOURCE_GROUP" --query "oidcIssuerProfile.issuerUrl" --output tsv) + +# Create federated identity credential +az identity federated-credential create --name "federated-ns-$NAMESPACE" \ + --identity-name "$USER_ASSIGNED_IDENTITY_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --issuer "$AKS_OIDC_ISSUER" \ + --subject system:serviceaccount:"$NAMESPACE":"$SERVICE_ACCOUNT_NAME" \ + --audience api://AzureADTokenExchange +``` + +### Install Key Vault Secrets using Helm Chart + +Using the [ci-keyvault-secrets](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure/-/tree/master/ci-keyvault-secrets?ref_type=heads) Helm chart in the helm-charts-azure repo, install the secrets for Elasticsearch. + +```bash +# Azure Configuration +TENANT_ID="<tenant-id>" +CLIENT_ID="<user-assigned-client-id>" +KEYVAULT_NAME="<keyvault-name>" + +# Create Values File +cat << EOF > elastic-keyvault-values.yaml +azure: + tenantId: ${TENANT_ID} + clientId: ${CLIENT_ID} + keyvaultUri: https://${KEYVAULT_NAME}.vault.azure.net/ + keyvaultName: ${KEYVAULT_NAME} +identity: true + +secrets: + - secretName: elasticsearch-credentials + data: + - key: elastic-username + vaultSecret: elastic-search-username + - key: elastic-password + vaultSecret: elastic-search-password + - key: elastic-key + vaultSecret: elastic-search-key +EOF + +# Install Helm Chart +NAMESPACE=elastic-search +helm upgrade --install keyvault-secrets . -f elastic-keyvault-values.yaml -n ${NAMESPACE} --create-namespace +``` + + +### Install Certificate Manager using Helm Chart + +Install cert-manager using Helm + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.16.3 \ + --set crds.enabled=true +``` + +Deploy the ClusterIssuers + +```bash +ADMIN_EMAIL=user@gmail.com + +cat << EOF | kubectl apply -f - +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: ${ADMIN_EMAIL} + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: webapprouting.kubernetes.azure.com +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: ${ADMIN_EMAIL} + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - http01: + ingress: + class: webapprouting.kubernetes.azure.com +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-cluster-issuer +spec: + selfSigned: {} +EOF +``` + + +### Configure Cluster for External DNS + +#### Create a User-Assigned Managed Identity: + +```bash +IDENTITY_NAME="externaldns-identity" +IDENTITY_RESOURCE_GROUP="$RESOURCE_GROUP" +az identity create --name $IDENTITY_NAME --resource-group $IDENTITY_RESOURCE_GROUP +``` + +#### Assign Permissions + +```bash +DNS_ZONE_RESOURCE_GROUP="<dns-zone-resource-group>" +DNS_ZONE_NAME="<dns-zone-name>" +DNS_ZONE_ID=$(az network dns zone show --name $DNS_ZONE_NAME --resource-group $DNS_ZONE_RESOURCE_GROUP --query "id" --output tsv) +az role assignment create --role "DNS Zone Contributor" --assignee "$IDENTITY_CLIENT_ID" --scope "$DNS_ZONE_ID" +az role assignment create --role "Reader" --assignee "$IDENTITY_CLIENT_ID" --scope "/subscriptions/$SUBSCRIPTION/resourceGroups/$DNS_ZONE_RESOURCE_GROUP" +``` + +#### Create a Federated Identity Credential: + +```bash +NAMESPACE="external-dns" +SERVICE_ACCOUNT_NAME="workload-identity-sa" + +kubectl create namespace $NAMESPACE + +cat << EOF | kubectl apply -f - +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ${SERVICE_ACCOUNT_NAME} + namespace: ${NAMESPACE} + labels: + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: "${IDENTITY_CLIENT_ID}" + azure.workload.identity/tenant-id: "${TENANT_ID}" +EOF + +# Retrieve AKS OIDC issuer +AKS_OIDC_ISSUER=$(az aks show --name "$CLUSTER_NAME" --resource-group "$RESOURCE_GROUP" --query "oidcIssuerProfile.issuerUrl" --output tsv) + +# Create federated identity credential +az identity federated-credential create --name "federated-ns-$NAMESPACE" \ + --identity-name "$USER_ASSIGNED_IDENTITY_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --issuer "$AKS_OIDC_ISSUER" \ + --subject system:serviceaccount:"$NAMESPACE":"$SERVICE_ACCOUNT_NAME" \ + --audience api://AzureADTokenExchange +``` + +#### Install External DNS using Helm Chart + +```bash +# Set DNS Zone variables +DNS_ZONE_NAME="<your-dns-zone>" # e.g., example.com +DNS_ZONE_RESOURCE_GROUP="<dns-zone-resource-group>" # The resource group containing your DNS zone + +# Create values file for ExternalDNS +cat << EOF > external-dns-values.yaml +provider: azure +policy: sync + +azure: + useWorkloadIdentityExtension: true + tenantId: ${TENANT_ID} + subscriptionId: $(az account show --query id -o tsv) + resourceGroup: ${DNS_ZONE_RESOURCE_GROUP} + +txtOwnerId: external-dns +logLevel: info + +extraArgs: + txt-wildcard-replacement: wildcard + +podLabels: + azure.workload.identity/use: "true" + +serviceAccount: + create: false + name: ${SERVICE_ACCOUNT_NAME} + annotations: + azure.workload.identity/client-id: ${IDENTITY_CLIENT_ID} + +resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi + +domainFilters: + - ${DNS_ZONE_NAME} +EOF + +# Install ExternalDNS using Helm +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update + +helm upgrade --install external-dns bitnami/external-dns \ + --namespace ${NAMESPACE} \ + --values external-dns-values.yaml + +# Cleanup values file +rm external-dns-values.yaml +``` + + +### Install Elastic Cloud Kubernetes + +```bash +kubectl create -f https://download.elastic.co/downloads/eck/2.16.1/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.16.1/operator.yaml + +STORAGE_ACCOUNT_NAME="<storage-account-name>" +CONTAINER_NAME="elasticsearch-snapshots" + +cat << EOF > elasticsearch-values.yaml +azure: + storageAccountName: "$STORAGE_ACCOUNT_NAME" + storageAccountContainer: "$CONTAINER_NAME" +EOF + +NAMESPACE=elastic-search +helm upgrade --install elasticsearch . -f elasticsearch-values.yaml -n ${NAMESPACE} +``` + + +### Install Elastic Search Instance using Helm Chart + +Using the [ci-elastic-search](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure/-/tree/master/ci-elastic-search?ref_type=heads) Helm chart in the helm-charts-azure repo, install the secrets for Elasticsearch. + +```bash +ES_DNS_NAME="<fully-qualified-domain-name>" +KB_DNS_NAME="<fully-qualified-domain-name>" + +# Create values file for ExternalDNS +cat << EOF > elasticsearch-values.yaml +fullnameOverride: elastic-search + +storage: + size: "4Gi" + class: "managed-premium" + +resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "3Gi" + +replicaCount: 1 + +azure: + storageAccountName: ${STORAGE_ACCOUNT_NAME} + storageAccountContainer: ${CONTAINER_NAME} + +externalDNS: + clusterIssuer: "letsencrypt-prod" + certificateSecretName: "elastic-search-tls" + ingressClassName: "webapprouting.kubernetes.azure.com" + elasticsearchHostname: "${ES_DNS_NAME}" + kibanaHostname: "${KB_DNS_NAME}" + certificates: + elasticsearch: + secretName: "elastic-search-es-tls" + kibana: + secretName: "elastic-search-kb-tls" +EOF + +# Install Helm Chart +NAMESPACE=elastic-search +helm upgrade --install elastic-search . -f elasticsearch-values.yaml -n ${NAMESPACE} +``` diff --git a/infra/scripts/common_prepare.sh b/infra/scripts/common_prepare.sh old mode 100644 new mode 100755 index 8550d1b3ac2346e4a55d30e3c58ba8343d4fb5f5..a5d1e7610049d2dca842156b62b5081795563922 --- a/infra/scripts/common_prepare.sh +++ b/infra/scripts/common_prepare.sh @@ -70,10 +70,13 @@ if [ -z $AZURE_AKS_USER ]; then fi - - ############################### -## FUNCTIONS ## +## WSL DETECTION ## +############################### +if [ -f /proc/version ] && grep -qi microsoft /proc/version; then + export MSYS_NO_PATHCONV=1 +fi + ############################### function CreateResourceGroup() { # Required Argument $1 = RESOURCE_GROUP @@ -102,6 +105,7 @@ function CreateResourceGroup() { tput setaf 3; echo "Resource Group $1 already exists."; tput sgr0 fi } + function CreateTfPrincipal() { # Required Argument $1 = PRINCIPAL_NAME # Required Argument $2 = VAULT_NAME @@ -116,7 +120,7 @@ function CreateTfPrincipal() { if [ "$_result" == "" ] then - PRINCIPAL_SECRET=$(az ad sp create-for-rbac \ + PRINCIPAL_SECRET=$(MSYS_NO_PATHCONV=1 az ad sp create-for-rbac \ --name $1 \ --role owner \ --scopes /subscriptions/${ARM_SUBSCRIPTION_ID} \ @@ -138,7 +142,7 @@ function CreateTfPrincipal() { --api $MS_GRAPH_API_GUID \ --api-permissions $OWNED_BY_GUID=Role \ -ojsonc) - + # MS Graph API Directory.Read.All PERMISSION_2=$(az ad app permission add \ --id $PRINCIPAL_ID \ @@ -160,6 +164,7 @@ function CreateTfPrincipal() { tput setaf 3; echo "Service Principal $1 already exists."; tput sgr0 fi } + function CreatePrincipal() { # Required Argument $1 = PRINCIPAL_NAME # Required Argument $2 = VAULT_NAME @@ -173,49 +178,47 @@ function CreatePrincipal() { local _result=$(az ad sp list --display-name $1 --query [].appId -otsv) if [ "$_result" == "" ] then - - PRINCIPAL_SECRET=$(az ad sp create-for-rbac \ - --name $1 \ - --skip-assignment \ - --role owner \ - --scopes /subscriptions/${ARM_SUBSCRIPTION_ID} \ - --query password -otsv) - - PRINCIPAL_ID=$(az ad sp list \ - --display-name $1 \ - --query [].appId -otsv) - - PRINCIPAL_OID=$(az ad sp list \ - --display-name $1 \ - --query [].id -otsv) - - MS_GRAPH_API_GUID="00000003-0000-0000-c000-000000000000" - AZURE_STORAGE_API_GUID="e406a681-f3d4-42a8-90b6-c2b029497af1" - - - # MS Graph API Directory.Read.All - PERMISSION_1=$(az ad app permission add \ - --id $PRINCIPAL_ID \ - --api $MS_GRAPH_API_GUID \ - --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role \ - -ojsonc) - - # AzureStorage API user_impersonation scope + PRINCIPAL_SECRET=$(az ad sp create-for-rbac \ + --name $1 \ + --role owner \ + --scopes /subscriptions/${ARM_SUBSCRIPTION_ID} \ + --query password -otsv) + + PRINCIPAL_ID=$(az ad sp list \ + --display-name $1 \ + --query [].appId -otsv) + + PRINCIPAL_OID=$(az ad sp list \ + --display-name $1 \ + --query [].id -otsv) + + MS_GRAPH_API_GUID="00000003-0000-0000-c000-000000000000" + AZURE_STORAGE_API_GUID="e406a681-f3d4-42a8-90b6-c2b029497af1" + + # MS Graph API Directory.Read.All + PERMISSION_1=$(az ad app permission add \ + --id $PRINCIPAL_ID \ + --api $MS_GRAPH_API_GUID \ + --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role \ + -ojsonc) + + # AzureStorage API user_impersonation scope PERMISSION_2=$(az ad app permission add \ --id $PRINCIPAL_ID \ --api $AZURE_STORAGE_API_GUID \ --api-permissions 03e0da56-190b-40ad-a80c-ea378c433f7f=Scope \ -ojsonc) - tput setaf 2; echo "Adding Information to Vault..." ; tput sgr0 - AddKeyToVault $2 "${1}-id" $PRINCIPAL_ID - AddKeyToVault $2 "${1}-key" $PRINCIPAL_SECRET - AddKeyToVault $2 "${1}-oid" $PRINCIPAL_OID + tput setaf 2; echo "Adding Information to Vault..." ; tput sgr0 + AddKeyToVault $2 "${1}-id" $PRINCIPAL_ID + AddKeyToVault $2 "${1}-key" $PRINCIPAL_SECRET + AddKeyToVault $2 "${1}-oid" $PRINCIPAL_OID else tput setaf 3; echo "Service Principal $1 already exists."; tput sgr0 fi } + function CreateADApplication() { # Required Argument $1 = APPLICATION_NAME # Required Argument $2 = VAULT_NAME @@ -228,29 +231,28 @@ function CreateADApplication() { local _result=$(az ad sp list --display-name $1 --query [].appId -otsv) if [ "$_result" == "" ] then + APP_SECRET=$(MSYS_NO_PATHCONV=1 az ad sp create-for-rbac \ + --name $1 \ + --query password -otsv) - APP_SECRET=$(az ad sp create-for-rbac \ - --name $1 \ - --skip-assignment \ - --query password -otsv) - - APP_ID=$(az ad sp list \ - --display-name $1 \ - --query [].appId -otsv) + APP_ID=$(az ad sp list \ + --display-name $1 \ + --query [].appId -otsv) - APP_OID=$(az ad sp list \ - --display-name $1 \ - --query [].id -otsv) + APP_OID=$(az ad sp list \ + --display-name $1 \ + --query [].id -otsv) - tput setaf 2; echo "Adding AD Application to Vault..." ; tput sgr0 - AddKeyToVault $2 "${1}-clientid" $APP_ID - AddKeyToVault $2 "${1}-secret" $APP_SECRET - AddKeyToVault $2 "${1}-oid" $APP_OID + tput setaf 2; echo "Adding AD Application to Vault..." ; tput sgr0 + AddKeyToVault $2 "${1}-clientid" $APP_ID + AddKeyToVault $2 "${1}-secret" $APP_SECRET + AddKeyToVault $2 "${1}-oid" $APP_OID else tput setaf 3; echo "AD Application $1 already exists."; tput sgr0 fi } + function CreateSSHKeysPassphrase() { # Required Argument $1 = SSH_USER # Required Argument $2 = KEY_NAME @@ -292,6 +294,7 @@ function CreateSSHKeysPassphrase() { AddKeyToVault $AZURE_VAULT "${2}-pub" "~/.ssh/osdu_${UNIQUE}/${2}.pub" "file" AddKeyToVault $AZURE_VAULT "${2}-passphrase" $PASSPHRASE } + function CreateSSHKeys() { # Required Argument $1 = SSH_USER # Required Argument $2 = KEY_NAME @@ -351,11 +354,12 @@ function CreateKeyVault() { local _vault=$(az keyvault list --resource-group $2 --query [].name -otsv 2>/dev/null) if [ "$_vault" == "" ] then - OUTPUT=$(az keyvault create --name $1 --resource-group $2 --location $3 --enable-purge-protection true --query [].name -otsv) + OUTPUT=$(az keyvault create --name $1 --resource-group $2 --location $3 --query [].name -otsv) else tput setaf 3; echo "Key Vault $1 already exists."; tput sgr0 fi } + function CreateStorageAccount() { # Required Argument $1 = STORAGE_ACCOUNT # Required Argument $2 = RESOURCE_GROUP @@ -389,6 +393,7 @@ function CreateStorageAccount() { tput setaf 3; echo "Storage Account $1 already exists."; tput sgr0 fi } + function GetStorageAccountKey() { # Required Argument $1 = STORAGE_ACCOUNT # Required Argument $2 = RESOURCE_GROUP @@ -409,6 +414,7 @@ function GetStorageAccountKey() { --output tsv) echo ${_result} } + function CreateBlobContainer() { # Required Argument $1 = CONTAINER_NAME # Required Argument $2 = STORAGE_ACCOUNT @@ -445,6 +451,7 @@ function CreateBlobContainer() { tput setaf 3; echo "Storage Container $1 already exists."; tput sgr0 fi } + function AddKeyToVault() { # Required Argument $1 = KEY_VAULT # Required Argument $2 = SECRET_NAME diff --git a/infra/scripts/keda_upgrade_and_host_encryption.sh b/infra/scripts/keda_upgrade_and_host_encryption.sh old mode 100644 new mode 100755