README.md 12.2 KB
Newer Older
1
2
# Helm Installation Instructions

Daniel Scholl's avatar
Daniel Scholl committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
__CLI Login__

Login to Azure CLI using the OSDU Environment Service Principal.

```bash
# This logs your local Azure CLI in using the configured service principal.
az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
```

__Helm Values__

Create the helm chart values file necessary to install charts.

- Download [helm-config.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/helm-config.yaml), which will configure OSDU on Azure.

  `wget https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/helm-config.yaml -O config.yaml`

- Edit the newly downloaded [config.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/helm-config.yaml) and fill out the required sections `azure`, `ingress` and `istio`.

Daniel Scholl's avatar
Daniel Scholl committed
22
23


Daniel Scholl's avatar
Daniel Scholl committed
24
25
26
27
28
```bash
# Setup Variables
ADMIN_EMAIL="<your_cert_admin>"     # ie: admin@email.com
DNS_HOST="<your_ingress_hostname>"  # ie: osdu.contoso.com

Jason's avatar
Jason committed
29
GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
Daniel Scholl's avatar
Daniel Scholl committed
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)

# Translate Values File
cat > config.yaml << EOF
# This file contains the essential configs for the osdu on azure helm chart
global:

  # Service(s) Replica Count
  replicaCount: 2

  ################################################################################
  # Specify the azure environment specific values
  #
  azure:
    tenant: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/tenant-id --query value -otsv)
    subscription: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/subscription-id --query value -otsv)
    resourcegroup: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-cr --query value -otsv)-rg
    identity: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-cr --query value -otsv)-osdu-identity
    identity_id: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/osdu-identity-id --query value -otsv)
    keyvault: $ENV_VAULT
    appid: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/aad-client-id --query value -otsv)

  ################################################################################
  # Specify the Ingress Settings
  # DNS Hostname for thet Gateway
  # Admin Email Address to be notified for SSL expirations
  # Lets Encrypt SSL Server
  #     https://acme-staging-v02.api.letsencrypt.org/directory  --> Staging Server
  #     https://acme-v02.api.letsencrypt.org/directory --> Production Server
  #
  ingress:
    hostname: $DNS_HOST
    admin: $ADMIN_EMAIL
    sslServer: https://acme-v02.api.letsencrypt.org/directory  # Production

  ################################################################################
  # Specify the istio specific values
  # based64 encoded username and password
  #
  istio:
Jason's avatar
Jason committed
70
71
    username: $(az keyvault secret show --id https://${COMMON_VAULT}.vault.azure.net/secrets/istio-username --query value -otsv)
    password: $(az keyvault secret show --id https://${COMMON_VAULT}.vault.azure.net/secrets/istio-password --query value -otsv)
Daniel Scholl's avatar
Daniel Scholl committed
72
73
EOF
```
74
75
76
77
78
79
80
81
82
83
84
85
Create the helm chart values file necessary to install airflow charts.

- Download [helm-config.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml), which will configure OSDU on Azure.

  `wget https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml -O config_airflow.yaml`

- Edit the newly downloaded [config_airflow.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml) and fill out the required sections `externalDatabase` and `externalRedis`.



```bash
# Setup Variables
86
87
88
BRANCH="master"
TAG="latest"

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)

# Translate Values File
cat > config_airflow.yaml << EOF
# This file contains the essential configs for the osdu airflow on azure helm chart
appinsightstatsd:
  aadpodidbinding: "osdu-identity"
airflowLogin:
  name: admin
airflow:
  airflow:
    image:
      repository: apache/airflow
      tag: 1.10.12-python3.6
      pullPolicy: IfNotPresent
      pullSecret: ""
    config:
      AIRFLOW__SCHEDULER__STATSD_ON: "True"
      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
      ## Enable for Debug purpose
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
      AIRFLOW__API__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
      AIRFLOW__CORE__REMOTE_LOGGING: "True"
      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log"
      AIRFLOW__CELERY__SSL_ACTIVE: "True"
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
124
      AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    extraEnv:
      - name: AIRFLOW__CORE__FERNET_KEY
        valueFrom:
          secretKeyRef:
            name: airflow
            key: fernet-key
      - name: AIRFLOW_CONN_AZ_LOG
        valueFrom:
          secretKeyRef:
            name: airflow
            key: remote-log-connection
    extraConfigmapMounts:
      - name: remote-log-config
        mountPath: /opt/airflow/config
        configMap: airflow-remote-log-config
        readOnly: true
    extraPipPackages: [
        "flask-bcrypt",
        "apache-airflow[statsd]",
        "apache-airflow[kubernetes]",
        "apache-airflow-backport-providers-microsoft-azure"
    ]
    extraVolumeMounts:
      - name: azure-keyvault
        mountPath: "/mnt/azure-keyvault"
        readOnly: true
151
152
153
      - name: dags-data
        mountPath: /opt/airflow/plugins
        subPath: plugins
154
155
156
157
158
159
160
161
162
163
164
165
    extraVolumes:
      - name: azure-keyvault
        csi:
          driver: secrets-store.csi.k8s.io
          readOnly: true
          volumeAttributes:
            secretProviderClass: azure-keyvault
  dags:
    installRequirements: true
    persistence:
      enabled: true
      existingClaim: airflowdagpvc
166
      subPath: "dags"
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
  scheduler:
    podLabels:
      aadpodidbinding: "osdu-identity"
    variables: |
      {}
  web:
    podLabels:
      aadpodidbinding: "osdu-identity"
    baseUrl: "http://localhost/airflow"
  workers:
    podLabels:
      aadpodidbinding: "osdu-identity"
  flower:
    enabled: false
  postgresql:
    enabled: false
  externalDatabase:
    type: postgres
    ## Azure PostgreSQL Database username, formatted as {username}@{hostname}
    user:  osdu_admin@$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"
    ## Azure PostgreSQL Database host
    host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg.postgres.database.azure.com
    port: 5432
    properties: "?sslmode=require"
    database: airflow
  redis:
    enabled: false
  externalRedis:
    ## Azure Redis Cache host
    host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-cache.redis.cache.windows.net
    port: 6380
    passwordSecret: "redis"
    passwordSecretKey: "redis-password"

203
204
205
206
image:
  repository: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv).azurecr.io
  branch: $BRANCH
  tag: $TAG
207
208
EOF
```
Daniel Scholl's avatar
Daniel Scholl committed
209
210
211
212

__Clone Service Repositories__

Manually ensure all the services have been cloned to your machine.
213
214

```bash
Daniel Scholl's avatar
Daniel Scholl committed
215
216
217
218
219
220
221
222
223
SRC_DIR="<ROOT_PATH_TO_SOURCE>" #  $HOME/source/osdu/osdu-gitlab

git clone https://community.opengroup.org/osdu/platform/system/partition.git $SRC_DIR/partition
git clone https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure.git $SRC_DIR/entitlements-azure
git clone https://community.opengroup.org/osdu/platform/security-and-compliance/legal.git $SRC_DIR/legal
git clone https://community.opengroup.org/osdu/platform/system/storage.git $SRC_DIR/storage
git clone https://community.opengroup.org/osdu/platform/system/indexer-queue.git $SRC_DIR/indexer-queue
git clone https://community.opengroup.org/osdu/platform/system/indexer-service.git $SRC_DIR/indexer-service
git clone https://community.opengroup.org/osdu/platform/system/search-service.git $SRC_DIR/search-service
224
225
226
```


Daniel Scholl's avatar
Daniel Scholl committed
227
228
229
230
__Kubernetes API Access__

> Optional

Daniel Scholl's avatar
Daniel Scholl committed
231
It can often be helpful to be able to retrieve the cluster context and execute queries directly against the Kubernetes API.
Daniel Scholl's avatar
Daniel Scholl committed
232
233

```bash
Jason's avatar
Jason committed
234
BASE_NAME=$(az group list --query "[?contains(name, 'sr${UNIQUE}')].name" -otsv |grep -v MC | rev | cut -c 3- | rev)
Daniel Scholl's avatar
Daniel Scholl committed
235
236
237
238
239

az aks get-credentials -n ${BASE_NAME}aks -g ${BASE_NAME}rg
```


Daniel Scholl's avatar
Daniel Scholl committed
240
241
242
__Helm Manifests__

Manually extract the manifests from the helm charts to your Flux Repo Directory.
243
244

```bash
Daniel Scholl's avatar
Daniel Scholl committed
245
246
INFRA_SRC="$SRC_DIR/infra-azure-provisioning"
FLUX_SRC="$INFRA_SRC/k8-gitops-manifests"
247
248
249
BRANCH="master"
TAG="latest"

Daniel Scholl's avatar
Daniel Scholl committed
250
251
252
# Setup the Flux Directory
mkdir -p ${FLUX_SRC}/providers/azure/hld-registry

253
254
255
# Extract manifests from the common osdu chart.
helm template osdu-flux ${INFRA_SRC}/charts/osdu-common -f ${INFRA_SRC}/charts/config.yaml > ${FLUX_SRC}/providers/azure/hld-registry/osdu-common.yaml

Daniel Scholl's avatar
Daniel Scholl committed
256
257
258
259
260
261
262
# Commit and Checkin to Deploy
(cd $FLUX_SRC \
  && git switch -c $UNIQUE \
  && git add ${FLUX_SRC}/providers/azure/hld-registry/osdu-common.yaml \
  && git commit -m "Initialize Common Chart" \
  && git push origin $UNIQUE)

Daniel Scholl's avatar
Daniel Scholl committed
263
# Extract manifests from the istio charts.
264
265
266
267
helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio -f ${INFRA_SRC}/charts/config.yaml > ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio.yaml

helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio-auth -f ${INFRA_SRC}/charts/config.yaml > ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio-auth.yaml

Daniel Scholl's avatar
Daniel Scholl committed
268
269
270
# Commit and Checkin to Deploy
(cd $FLUX_SRC \
  && git switch $UNIQUE \
Daniel Scholl's avatar
Daniel Scholl committed
271
  && git add ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio.yaml \
Daniel Scholl's avatar
Daniel Scholl committed
272
273
274
275
  && git add ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio-auth.yaml \
  && git commit -m "Initialize Istio Auth Chart" \
  && git push origin $UNIQUE)

276
277
278
279
280
281
282
283
284
285
286
287
# Publish Docker Images for airflow components
CONTAINER_REGISTRY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv)
az acr login -n $CONTAINER_REGISTRY
for SERVICE in airflow-function;
do
  cd ${INFRA_SRC}/source/$SERVICE
  IMAGE=$CONTAINER_REGISTRY.azurecr.io/$SERVICE-$BRANCH:$(TAG)
  docker build -t $IMAGE .
  docker push $IMAGE
done

# Installing PyYaml required for airflow
288
pip3 install -U PyYAML
289

290
291
292
293
294
295
296
297
298
299
# Extract manifests from the airflow charts.
helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_airflow.yaml | python3 ${INFRA_SRC}/charts/airflow/scripts/add-namespace.py > ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml

# Commit and Checkin to Deploy
(cd $FLUX_SRC \
  && git switch $UNIQUE \
  && git add ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml \
  && git commit -m "Initialize Airflow Chart" \
  && git push origin $UNIQUE)

Daniel Scholl's avatar
Daniel Scholl committed
300
# Extract manifests from each service chart.
Daniel Scholl's avatar
Daniel Scholl committed
301
for SERVICE in partition entitlements-azure legal storage indexer-queue indexer-service search-service;
Daniel Scholl's avatar
Daniel Scholl committed
302
303
304
305
306
307
308
309
310
311
do
  helm template $SERVICE ${SRC_DIR}/$SERVICE/devops/azure/chart --set image.branch=$BRANCH --set image.tag=$TAG > ${FLUX_SRC}/providers/azure/hld-registry/$SERVICE.yaml
done

# Commit and Checkin to Deploy
(cd $FLUX_SRC \
  && git switch $UNIQUE \
  && git add ${FLUX_SRC}/providers/azure/hld-registry/*.yaml \
  && git commit -m "Adding OSDU Services" \
  && git push origin $UNIQUE)
312
```