Commit 81620935 authored by Daniel Scholl's avatar Daniel Scholl
Browse files

Enable Manifest Ingestion

parent e44e6560
......@@ -3,6 +3,7 @@
# Current Master
__Bug Fixes__
- [Feature 125 - Enable Manifest Ingestion](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/125)
- [Bug 119 - Add Airflow python package `python-keycloak`](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/119)
# v0.5 (2021-2-11)
......
......@@ -221,7 +221,7 @@ cp .envrc .envrc_${UNIQUE}
```
## Configure Back Up
Back is enabled by default. To set the backup policies, utilize the script
Back is enabled by default. To set the backup policies, utilize the script
[here](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/tree/master/tools).
The script should be run whenever you bring up a Resource Group in your deployment.
......
......@@ -62,3 +62,27 @@ az storage file upload-batch \
--source ${PROJECT_FOLDER} \
--pattern ${SOURCE_FOLDER}
```
_Ingest Manifest DAGS_
```bash
FILE_SHARE="airflowdags"
PROJECT_FOLDER=$(realpath ../ingestion-dags/src)
GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
az storage file upload-batch \
--account-name $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv) \
--account-key $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv) \
--destination $FILE_SHARE \
--source ${PROJECT_FOLDER} \
--pattern "*.ini"
az storage file upload-batch \
--account-name $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv) \
--account-key $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv) \
--destination $FILE_SHARE \
--source ${PROJECT_FOLDER} \
--pattern "*.py"
```
This diff is collapsed.
......@@ -38,11 +38,53 @@ grant_type=refresh_token
@ENDPOINT = https://{{OSDU_HOST}}
@INGESTION_WORKLFOW_HOST = {{ENDPOINT}}/api/workflow/v1
@data_partition_id = opendes
@workflow_name = Default_ingest
# -----------------------
# API: Ingestion Workflow
# -----------------------
##### NOT IMPLEMENTED YET
###
# @name SwaggerUI
GET {{INGESTION_WORKLFOW_HOST}}/swagger-ui.html
# @name getAllWorkflow
GET {{INGESTION_WORKLFOW_HOST}}/workflow
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
###
# @name createWorkflow
POST {{INGESTION_WORKLFOW_HOST}}/workflow
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
{
"workflowName": "{{workflow_name}}",
"description": "Test Register a Workflow",
"registrationInstructions": {
"active": true,
"concurrentWorkflowRun": 5,
"concurrentTaskRun": 5,
"workflowDetailContent": ""
}
}
###
# @name getWorkflow
GET {{INGESTION_WORKLFOW_HOST}}/workflow/{{workflow_name}}
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
##### NOT IMPLEMENTED FULLY YET
###
# @name deleteWorkflow
DELETE {{INGESTION_WORKLFOW_HOST}}/workflow/{{workflow_name}}
Authorization: Bearer {{access_token}}
Content-Type: application/json
data-partition-id: {{data_partition_id}}
......@@ -57,10 +57,12 @@ aad_client_id="${ENV_APP_ID}"
appinsights_key="${ENV_APPINSIGHTS_KEY}"
cosmosdb_database="${COSMOS_DB_NAME}"
entitlements_service_api_key="${API_KEY}"
entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1/"
entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1"
airflow_url="https://${ENV_HOST}/airflow"
airflow_username="admin"
airflow_password="${AIRFLOW_ADMIN_PASSWORD}"
airflow_password="${ENV_AIRFLOW_PASSWORD}"
AIRFLOW_STORAGE_ACCOUNT_NAME=${ENV_FILESHARE_STORAGE}
AIRFLOW_STORAGE_ACCOUNT_KEY=${ENV_FILESHARE_STORAGE_KEY}
LOG_PREFIX="workflow"
server_port="8085"
azure_istioauth_enabled="true"
......@@ -128,6 +130,9 @@ export ENV_ELASTIC_HOST=$ENV_ELASTIC_HOST
export ENV_ELASTIC_PORT=$ENV_ELASTIC_PORT
export ENV_ELASTIC_USERNAME=$ENV_ELASTIC_USERNAME
export ENV_ELASTIC_PASSWORD=$ENV_ELASTIC_PASSWORD
export ENV_AIRFLOW_PASSWORD=$ENV_AIRFLOW_PASSWORD
export ENV_FILESHARE_STORAGE=$ENV_FILESHARE_STORAGE
export ENV_FILESHARE_STORAGE_KEY=$ENV_FILESHARE_STORAGE_KEY
# ------------------------------------------------------------------------------------------------------
# LocalHost Run Settings
......@@ -140,10 +145,12 @@ export aad_client_id="${ENV_APP_ID}"
export appinsights_key="${ENV_APPINSIGHTS_KEY}"
export cosmosdb_database="${COSMOS_DB_NAME}"
export entitlements_service_api_key="${API_KEY}"
export entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1/"
export entitlements_service_endpoint="https://${ENV_HOST}/entitlements/v1"
export airflow_url="https://${ENV_HOST}/airflow"
export airflow_username="admin"
export airflow_password="${AIRFLOW_ADMIN_PASSWORD}"
export airflow_password="${ENV_AIRFLOW_PASSWORD}"
export AIRFLOW_STORAGE_ACCOUNT_NAME=${ENV_FILESHARE_STORAGE}
export AIRFLOW_STORAGE_ACCOUNT_KEY=${ENV_FILESHARE_STORAGE_KEY}
export LOG_PREFIX="workflow"
export server_port="8085"
export azure_istioauth_enabled="true"
......@@ -173,10 +180,12 @@ aad_client_id: "${ENV_APP_ID}"
appinsights_key: "${ENV_APPINSIGHTS_KEY}"
cosmosdb_database: "${COSMOS_DB_NAME}"
entitlements_service_api_key: "${API_KEY}"
entitlements_service_endpoint: "https://${ENV_HOST}/entitlements/v1/"
entitlements_service_endpoint: "https://${ENV_HOST}/entitlements/v1"
airflow_url: "https://${ENV_HOST}/airflow"
airflow_username: "admin"
airflow_password: "${AIRFLOW_ADMIN_PASSWORD}"
airflow_password: "${ENV_AIRFLOW_PASSWORD}"
AIRFLOW_STORAGE_ACCOUNT_NAME: "${ENV_FILESHARE_STORAGE}"
AIRFLOW_STORAGE_ACCOUNT_KEY: "${ENV_FILESHARE_STORAGE_KEY}"
LOG_PREFIX: "workflow"
server_port: "8085"
azure_istioauth_enabled: "true"
......@@ -207,4 +216,4 @@ WORKFLOW_HOST: "https://${ENV_HOST}/api/workflow/v1"
DEFAULT_DATA_PARTITION_ID_TENANT1: "${OSDU_TENANT}"
DOMAIN: "${COMPANY_DOMAIN}"
FINISHED_WORKFLOW_ID: "c80a2419-8527-4804-b96a-6b6444f0d361"
DEVTEST
\ No newline at end of file
DEVTEST
......@@ -29,4 +29,7 @@ ENV_ELASTIC_HOST="$(echo $(az keyvault secret show --id https://${ENV_VAULT}.vau
ENV_ELASTIC_PORT="9243"
ENV_ELASTIC_USERNAME="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${OSDU_TENANT}-elastic-username --query value -otsv)"
ENV_ELASTIC_PASSWORD="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${OSDU_TENANT}-elastic-password --query value -otsv)"
ENV_AIRFLOW_PASSWORD="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-admin-password --query value -otsv)"
API_KEY="OBSOLETE"
ENV_FILESHARE_STORAGE="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv)"
ENV_FILESHARE_STORAGE_KEY="$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv)"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment