diff --git a/helm/osdu-infra-onprem/README.md b/helm/osdu-infra-onprem/README.md index 1dc5c3ecae3b4605e9fabd6d06a7faf286efdabe..5aaab62b406cfdc7f12dd770570d0e280adcc8ac 100644 --- a/helm/osdu-infra-onprem/README.md +++ b/helm/osdu-infra-onprem/README.md @@ -215,27 +215,39 @@ List of arbitrary env variables. These variables must be set for Scheduler, Work | AIRFLOW\_VAR\_ANTHOS\_\_IMAGE\_\_SEGY\_TO\_VDS\_CONVERTER | anthos\_\_image\_\_segy\_to\_vds\_converter | {path-to-image}/segy-to-vds-conversion | | AIRFLOW\_VAR\_ANTHOS\_\_IMAGE\_\_SEGY\_TO\_ZGY\_CONVERTER | anthos\_\_image\_\_segy\_to\_zgy\_converter | {path-to-image}/segy-to-zgy-conversion | | AIRFLOW\_VAR\_ANTHOS\_\_IMAGE\_\_WITSML\_PARSER | anthos\_\_image\_\_witsml\_parser | {path-to-image}/witsml-parser | -| AIRFLOW\_VAR\_CORE\_\_CONFIG\_\_DATA\_PARTITION | core\_\_config\_\_data\_partition | anthos | -| AIRFLOW\_VAR\_CORE\_\_CONFIG\_\_SHOW\_SKIPPED\_IDS | core\_\_config\_\_show\_skipped\_ids | true | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_DATASET\_\_URL | core\_\_service\_\_dataset\_\_url | {schema}://{service-host}/api/dataset/v1 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_ENTITLEMENTS\_\_URL | core\_\_service\_\_entitlements\_\_url | {schema}://{service-host}/api/entitlements/v2 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_FILE\_\_HOST | core\_\_service\_\_file\_\_host | {schema}://{service-host}/api/file | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_FILE\_\_URL | core\_\_service\_\_file\_\_url | {schema}://{service-host}/api/file/v2 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_LEGAL\_\_URL | core\_\_service\_\_legal\_\_url | {schema}://{service-host}/api/legal/v1 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_MINIO\_\_URL | core\_\_service\_\_minio\_\_url | {minio-url} | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_PARTITION\_\_URL | core\_\_service\_\_partition\_\_url | {schema}://{service-host}/api/partition/v1/ | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SCHEMA\_\_HOST | core\_\_service\_\_schema\_\_host | {service-host} | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SCHEMA\_\_URL | core\_\_service\_\_schema\_\_url | {schema}://{service-host}/api/schema-service/v1 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEARCH\_\_HOST | core\_\_service\_\_search\_\_host | {service-host} | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEARCH\_\_URL | core\_\_service\_\_search\_\_url | {schema}://{service-host}/api/search/v2/query | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEISMIC\_\_URL | core\_\_service\_\_seismic\_\_url | {schema}://{service-host}/api/seismic-store/v3 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_STORAGE\_\_HOST | core\_\_service\_\_storage\_\_host | {service-host} | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_STORAGE\_\_URL | core\_\_service\_\_storage\_\_url | {schema}://{service-host}/api/storage/v2 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_UNIT\_\_URL | core\_\_service\_\_unit\_\_url | {schema}://{service-host}/api/unit/v2 | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_WORKFLOW\_\_HOST | core\_\_service\_\_workflow\_\_host | {schema}://{service-host}/api/workflow | -| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_WORKFLOW\_\_URL | core\_\_service\_\_workflow\_\_url | {schema}://{service-host}/api/workflow/v1/ | -| AIRFLOW\_VAR\_ANTHOS__PARSERS_K8S_NAMESPACE | anthos__parsers_k8s_namespace | default | -| AIRFLOW\_VAR\_CORE__INGESTION__BATCH_SAVE_ENABLED | core__ingestion__batch_save_enabled | true | +| AIRFLOW\_VAR\_ANTHOS\_\_DATA\_PARTITION\_ID | anthos\_\_data\_partition\_id | from {{ .Values.bootstrap.infra.secret.dataPartitionId }} | +| AIRFLOW\_VAR\_ANTHOS\_\_ZGY\_INGESTION\_LIMIT\_CPU | anthos\_\_zgy\_ingestion\_limit\_cpu | 1 | +| AIRFLOW\_VAR\_ANTHOS\_\_ZGY\_INGESTION\_LIMIT\_MEMORY | anthos\_\_zgy\_ingestion\_limit\_memory | 8Gi | +| AIRFLOW\_VAR\_ANTHOS\_\_ZGY\_INGESTION\_REQUEST\_CPU | anthos\_\_zgy\_ingestion\_request\_cpu | 100m | +| AIRFLOW\_VAR\_ANTHOS\_\_WITSML\_INGESTION\_LIMIT\_CPU | anthos\_\_witsml\_ingestion\_limit\_cpu | 1 | +| AIRFLOW\_VAR\_ANTHOS\_\_WITSML\_INGESTION\_LIMIT\_MEMORY | anthos\_\_witsml\_ingestion\_limit\_memory | 8Gi | +| AIRFLOW\_VAR\_ANTHOS\_\_WITSML\_INGESTION\_REQUEST\_CPU | anthos\_\_witsml\_ingestion\_request\_cpu | 100m | +| AIRFLOW\_VAR\_CORE\_\_CONFIG\_\_DATA\_PARTITION | core\_\_config\_\_data\_partition | from {{ .Values.bootstrap.infra.secret.dataPartitionId }} | +| AIRFLOW\_VAR\_CORE\_\_CONFIG\_\_SHOW\_SKIPPED\_IDS | core\_\_config\_\_show\_skipped\_ids | true | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_DATASET\_\_HOST | core\_\_service\_\_dataset\_\_host | http://dataset | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_DATASET\_\_URL | core\_\_service\_\_dataset\_\_url | http://dataset/api/dataset/v1 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_ENTITLEMENTS\_\_HOST | core\_\_service\_\_entitlements\_\_host | http://entitlements | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_ENTITLEMENTS\_\_URL | core\_\_service\_\_entitlements\_\_url | http://entitlements/api/entitlements/v2 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_FILE\_\_HOST | core\_\_service\_\_file\_\_host | http://file | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_FILE\_\_URL | core\_\_service\_\_file\_\_url | http://entitlements/api/file/v2 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_LEGAL\_\_HOST | core\_\_service\_\_legal\_\_host | http://legal | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_LEGAL\_\_URL | core\_\_service\_\_legal\_\_url | http://legal/api/legal/v1 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_MINIO\_\_URL | core\_\_service\_\_minio\_\_url | http://minio:9000 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_PARTITION\_\_HOST | core\_\_service\_\_partition\_\_url | http://partition/api/partition/v1/ | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_PARTITION\_\_URL | core\_\_service\_\_partition\_\_url | http://partition/api/partition/v1/ | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SCHEMA\_\_HOST | core\_\_service\_\_schema\_\_host | http://schema | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SCHEMA\_\_URL | core\_\_service\_\_schema\_\_url | http://schema/api/schema-service/v1 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEARCH\_\_HOST | core\_\_service\_\_search\_\_host | http://search | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEARCH\_\_URL | core\_\_service\_\_search\_\_url | http://search/api/search/v2/query | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEISMIC\_\_HOST | core\_\_service\_\_seismic\_\_host | http://seismic-store | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_SEISMIC\_\_URL | core\_\_service\_\_seismic\_\_url | http://seismic-store/api/seismic-store/v3 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_STORAGE\_\_HOST | core\_\_service\_\_storage\_\_host | http://storage | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_STORAGE\_\_URL | core\_\_service\_\_storage\_\_url | http://storage/api/storage/v2 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_UNIT\_\_HOST | core\_\_service\_\_unit\_\_host | http://unit | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_UNIT\_\_URL | core\_\_service\_\_unit\_\_url | http://unit/api/unit/v2 | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_WORKFLOW\_\_HOST | core\_\_service\_\_workflow\_\_host | http://workflow | +| AIRFLOW\_VAR\_CORE\_\_SERVICE\_\_WORKFLOW\_\_URL | core\_\_service\_\_workflow\_\_url | http://workflow/api/workflow/v1/ | +| AIRFLOW\_VAR\_CORE__INGESTION__BATCH_SAVE_ENABLED | core__ingestion__batch_save_enabled | true | | AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION | -- | "False" | | CLOUD_PROVIDER| -- | anthos | | KEYCLOAK_AUTH| -- | yes | @@ -244,8 +256,8 @@ List of arbitrary env variables. These variables must be set for Scheduler, Work | AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID | --| minio | | AIRFLOW__LOGGING__REMOTE_LOGGING | -- | **"True"** | | AIRFLOW__WEBSERVER__EXPOSE_CONFIG | -- | **"True"** | -| KEYCLOAK_AUTH_URL | -- | {schema}://{host}/realms/master/protocol/openid-connect/token | -| CSV_PARSER_KEYCLOAK_AUTH_URL | -- | {schema}://{host}/realms/master | +| KEYCLOAK_AUTH_URL | -- | http://{host}/realms/master/protocol/openid-connect/token | +| CSV_PARSER_KEYCLOAK_AUTH_URL | -- | http://{host}/realms/master | #### Secrets diff --git a/helm/osdu-infra-onprem/templates/airflow-infra.yaml b/helm/osdu-infra-onprem/templates/airflow-infra.yaml index ae2e3f8a493a4a538c077b46be8903c800055bc6..49f95345775f3493e4c3192af172a07df5c5cf47 100644 --- a/helm/osdu-infra-onprem/templates/airflow-infra.yaml +++ b/helm/osdu-infra-onprem/templates/airflow-infra.yaml @@ -51,27 +51,40 @@ data: AIRFLOW_VAR_ANTHOS__IMAGE__SEGY_TO_VDS_CONVERTER: "gcr.io/osdu-anthos/segy-to-vds-conversion" AIRFLOW_VAR_ANTHOS__IMAGE__SEGY_TO_ZGY_CONVERTER: "gcr.io/osdu-anthos/segy-to-zgy-conversion" AIRFLOW_VAR_ANTHOS__IMAGE__WITSML_PARSER: "gcr.io/osdu-anthos/witsml-parser" + AIRFLOW_VAR_ANTHOS__DATA_PARTITION_ID: {{ .Values.bootstrap.infra.secret.dataPartitionId | quote }} + AIRFLOW_VAR_ANTHOS__ZGY_INGESTION_LIMIT_CPU: "1" + AIRFLOW_VAR_ANTHOS__ZGY_INGESTION_LIMIT_MEMORY: "8Gi" + AIRFLOW_VAR_ANTHOS__ZGY_INGESTION_REQUEST_CPU: "100m" + AIRFLOW_VAR_ANTHOS__WITSML_INGESTION_LIMIT_CPU: "1" + AIRFLOW_VAR_ANTHOS__WITSML_INGESTION_LIMIT_MEMORY: "8Gi" + AIRFLOW_VAR_ANTHOS__WITSML_INGESTION_REQUEST_CPU: "100m" AIRFLOW_VAR_CORE__CONFIG__DATA_PARTITION: {{ .Values.bootstrap.infra.secret.dataPartitionId | quote }} AIRFLOW_VAR_CORE__CONFIG__SHOW_SKIPPED_IDS: "true" + AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset" AIRFLOW_VAR_CORE__SERVICE__DATASET__URL: "http://dataset/api/dataset/v1" - AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: http://entitlements/api/entitlements/v2 - AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file/api/file" + AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__HOST: "http://entitlements" + AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements/api/entitlements/v2" + AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file" AIRFLOW_VAR_CORE__SERVICE__FILE__URL: "http://file/api/file/v2" + AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal" AIRFLOW_VAR_CORE__SERVICE__LEGAL__URL: "http://legal/api/legal/v1" - AIRFLOW_VAR_CORE__SERVICE__MINIO__URL: "http://minio" + AIRFLOW_VAR_CORE__SERVICE__MINIO__URL: "http://minio:9000" + AIRFLOW_VAR_CORE__SERVICE__PARTITION__HOST: "http://partition" AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition/api/partition/v1/" AIRFLOW_VAR_CORE__SERVICE__SCHEMA__HOST: "http://schema" AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema/api/schema-service/v1" AIRFLOW_VAR_CORE__SERVICE__SEARCH__HOST: "http://search" AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search/api/search/v2" + AIRFLOW_VAR_CORE__SERVICE__SEISMIC__HOST: "http://seismic-store" AIRFLOW_VAR_CORE__SERVICE__SEISMIC__URL: "http://seismic-store/api/seismic-store/v3" AIRFLOW_VAR_CORE__SERVICE__STORAGE__HOST: "http://storage" AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage/api/storage/v2" + AIRFLOW_VAR_CORE__SERVICE__UNIT__HOST: "http://unit" AIRFLOW_VAR_CORE__SERVICE__UNIT__URL: "http://unit/api/unit/v2" - AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow/api/workflow" + AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow" AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__URL: "http://workflow/api/workflow/v1" - AIRFLOW_VAR_ANTHOS__PARSERS_K8S_NAMESPACE: {{ .Release.Namespace }} AIRFLOW_VAR_CORE__INGESTION__BATCH_SAVE_ENABLED: "true" + AIRFLOW_VAR_CORE__INGESTION__BATCH_SAVE_SIZE: "250" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" CLOUD_PROVIDER: "anthos" KEYCLOAK_AUTH: "yes"