Merge branch 'manifest_ingest' into 'master'

Enable Manifest Ingestion

See merge request osdu/platform/deployment-and-operations/infra-azure-provisioning!222

.trufflehog-exclude

 CHANGELOG.md
+.*\.http$

CHANGELOG.md

@@ -3,6 +3,7 @@
 # Current Master
 
 __Bug Fixes__
+- [Feature 125 - Enable Manifest Ingestion](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/125)
 - [Bug 119 - Add Airflow python package `python-keycloak`](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/119)
 
 # v0.5 (2021-2-11)

README.md

@@ -221,7 +221,7 @@ cp .envrc .envrc_${UNIQUE}
 ```
 
 ## Configure Back Up
-Back is enabled by default. To set the backup policies, utilize the script 
+Back is enabled by default. To set the backup policies, utilize the script
 [here](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/tree/master/tools).
 The script should be run whenever you bring up a Resource Group in your deployment.
 

docs/configuration-data.md

@@ -62,3 +62,27 @@ az storage file upload-batch \
   --source ${PROJECT_FOLDER} \
   --pattern ${SOURCE_FOLDER}
 ```
+
+_Ingest Manifest DAGS_
+
+```bash
+FILE_SHARE="airflowdags"
+PROJECT_FOLDER=$(realpath ../ingestion-dags/src)
+
+GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
+ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
+
+az storage file upload-batch \
+  --account-name $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv) \
+  --account-key $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv) \
+  --destination $FILE_SHARE \
+  --source ${PROJECT_FOLDER} \
+  --pattern "*.ini"
+
+az storage file upload-batch \
+  --account-name $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv) \
+  --account-key $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv) \
+  --destination $FILE_SHARE \
+  --source ${PROJECT_FOLDER} \
+  --pattern "*.py"
+```

tools/rest/ingestion-workflow.http

@@ -38,11 +38,53 @@ grant_type=refresh_token
 @ENDPOINT = https://{{OSDU_HOST}}
 @INGESTION_WORKLFOW_HOST = {{ENDPOINT}}/api/workflow/v1
 @data_partition_id = opendes
+@workflow_name = Default_ingest
 
 # -----------------------
 # API: Ingestion Workflow
 # -----------------------
 
+##### NOT IMPLEMENTED YET
 ###
-# @name SwaggerUI
-GET {{INGESTION_WORKLFOW_HOST}}/swagger-ui.html
+# @name getAllWorkflow
+GET {{INGESTION_WORKLFOW_HOST}}/workflow
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+
+
+###
+# @name createWorkflow
+POST {{INGESTION_WORKLFOW_HOST}}/workflow
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+{
+  "workflowName": "{{workflow_name}}",
+  "description": "Test Register a Workflow",
+  "registrationInstructions": {
+    "active": true,
+    "concurrentWorkflowRun": 5,
+    "concurrentTaskRun": 5,
+    "workflowDetailContent": ""
+  }
+}
+
+
+###
+# @name getWorkflow
+GET {{INGESTION_WORKLFOW_HOST}}/workflow/{{workflow_name}}
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}
+
+
+##### NOT IMPLEMENTED FULLY YET
+###
+# @name deleteWorkflow
+DELETE {{INGESTION_WORKLFOW_HOST}}/workflow/{{workflow_name}}
+Authorization: Bearer {{access_token}}
+Content-Type: application/json
+data-partition-id: {{data_partition_id}}

tools/variables/ingestion-workflow.sh

@@ -57,10 +57,12 @@ aad_client_id="${ENV_APP_ID}"
 appinsights_key="${ENV_APPINSIGHTS_KEY}"
 cosmosdb_database="${COSMOS_DB_NAME}"
 entitlements_service_api_key="${API_KEY}"
-entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1/"
+entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1"
 airflow_url="https://${ENV_HOST}/airflow"
 airflow_username="admin"
-airflow_password="${AIRFLOW_ADMIN_PASSWORD}"
+airflow_password="${ENV_AIRFLOW_PASSWORD}"
+AIRFLOW_STORAGE_ACCOUNT_NAME=${ENV_FILESHARE_STORAGE}
+AIRFLOW_STORAGE_ACCOUNT_KEY=${ENV_FILESHARE_STORAGE_KEY}
 LOG_PREFIX="workflow"
 server_port="8085"
 azure_istioauth_enabled="true"
@@ -128,6 +130,9 @@ export ENV_ELASTIC_HOST=$ENV_ELASTIC_HOST
 export ENV_ELASTIC_PORT=$ENV_ELASTIC_PORT
 export ENV_ELASTIC_USERNAME=$ENV_ELASTIC_USERNAME
 export ENV_ELASTIC_PASSWORD=$ENV_ELASTIC_PASSWORD
+export ENV_AIRFLOW_PASSWORD=$ENV_AIRFLOW_PASSWORD
+export ENV_FILESHARE_STORAGE=$ENV_FILESHARE_STORAGE
+export ENV_FILESHARE_STORAGE_KEY=$ENV_FILESHARE_STORAGE_KEY
 
 # ------------------------------------------------------------------------------------------------------
 # LocalHost Run Settings
@@ -140,10 +145,12 @@ export aad_client_id="${ENV_APP_ID}"
 export appinsights_key="${ENV_APPINSIGHTS_KEY}"
 export cosmosdb_database="${COSMOS_DB_NAME}"
 export entitlements_service_api_key="${API_KEY}"
-export entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1/"
+export entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1"
 export airflow_url="https://${ENV_HOST}/airflow"
 export airflow_username="admin"
-export airflow_password="${AIRFLOW_ADMIN_PASSWORD}"
+export airflow_password="${ENV_AIRFLOW_PASSWORD}"
+export AIRFLOW_STORAGE_ACCOUNT_NAME=${ENV_FILESHARE_STORAGE}
+export AIRFLOW_STORAGE_ACCOUNT_KEY=${ENV_FILESHARE_STORAGE_KEY}
 export LOG_PREFIX="workflow"
 export server_port="8085"
 export azure_istioauth_enabled="true"
@@ -173,10 +180,12 @@ aad_client_id: "${ENV_APP_ID}"
 appinsights_key: "${ENV_APPINSIGHTS_KEY}"
 cosmosdb_database: "${COSMOS_DB_NAME}"
 entitlements_service_api_key: "${API_KEY}"
-entitlements_service_endpoint: "https://${ENV_HOST}/entitlements/v1/"
+entitlements_service_endpoint: "https://${ENV_HOST}/entitlements/v1"
 airflow_url: "https://${ENV_HOST}/airflow"
 airflow_username: "admin"
-airflow_password: "${AIRFLOW_ADMIN_PASSWORD}"
+airflow_password: "${ENV_AIRFLOW_PASSWORD}"
+AIRFLOW_STORAGE_ACCOUNT_NAME: "${ENV_FILESHARE_STORAGE}"
+AIRFLOW_STORAGE_ACCOUNT_KEY: "${ENV_FILESHARE_STORAGE_KEY}"
 LOG_PREFIX: "workflow"
 server_port: "8085"
 azure_istioauth_enabled: "true"
@@ -207,4 +216,4 @@ WORKFLOW_HOST: "https://${ENV_HOST}/api/workflow/v1"
 DEFAULT_DATA_PARTITION_ID_TENANT1: "${OSDU_TENANT}"
 DOMAIN: "${COMPANY_DOMAIN}"
 FINISHED_WORKFLOW_ID: "c80a2419-8527-4804-b96a-6b6444f0d361"
-DEVTEST
\ No newline at end of file
+DEVTEST

tools/variables/settings_environment.env

@@ -29,4 +29,7 @@ ENV_ELASTIC_HOST="$(echo $(az keyvault secret show --id https://${ENV_VAULT}.vau
 ENV_ELASTIC_PORT="9243"
 ENV_ELASTIC_USERNAME="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${OSDU_TENANT}-elastic-username --query value -otsv)"
 ENV_ELASTIC_PASSWORD="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${OSDU_TENANT}-elastic-password --query value -otsv)"
+ENV_AIRFLOW_PASSWORD="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-admin-password --query value -otsv)"
 API_KEY="OBSOLETE"
+ENV_FILESHARE_STORAGE="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv)"
+ENV_FILESHARE_STORAGE_KEY="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv)"