Merge branch 'master' of...

Merge branch 'master' of community.opengroup.org:osdu/platform/deployment-and-operations/infra-azure-provisioning

* 'master' of community.opengroup.org:osdu/platform/deployment-and-operations/infra-azure-provisioning:
  Adding script to configure the Backup for CosmosDB and Storage Account
  Adding ingestion service
  Added directions for loading schema's

README.md

@@ -192,7 +192,6 @@ __Azure AD Admin Consent__
 For more information on Azure identity and authorization, see the official Microsoft documentation [here](https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-permissions-and-consent).
 
 
-
 ## Elastic Search Setup
 
 Infrastructure requires a bring your own Elastic Search Instance of a version of 6.8.x with a valid https endpoint and the access information must now be stored in the Common KeyVault. The recommended method of Elastic Search is to use the [Elastic Cloud Managed Service from the Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/elastic.ec-azure?tab=Overview).
@@ -221,6 +220,10 @@ EOF
 cp .envrc .envrc_${UNIQUE}
 ```
 
+## Configure Back Up
+Back is enabled by default. To set the backup policies, utilize the script 
+[here](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/tree/master/tools).
+The script should be run whenever you bring up a Resource Group in your deployment.
 
 ## Configure Key Access in Manifest Repository
 
@@ -260,6 +263,8 @@ __Manual Installation__
 
 1. Register your partition with the Data Partition API by following the instructions [here](./tools/rest/README.md) to configure your IDE to make authenticated requests to your OSDU instance and send the API request located [here](./tools/rest/partition.http) (createPartition).
 
+1. Load Service Data following directions [here](./docs/service-data.md).
+
 
 
 __Automated Pipeline Installation__

charts/README.md

@@ -331,6 +331,7 @@ git clone https://community.opengroup.org/osdu/platform/system/schema-service.gi
 git clonehttps://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git $SRC_DIR/ingestion-workflow
 git clone https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git $SRC_DIR/seismic-store-service
 git clone https://community.opengroup.org:osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git $SRC_DIR/wellbore-domain-services
+git clone https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git $SRC_DIR/ingestion-service
 ```
 
 __Additional Manual Steps__
@@ -433,7 +434,8 @@ SERVICE_LIST="infra-azure-provisioning \
               register \
               notification \
               schema-service \
-              ingestion-workflow"
+              ingestion-workflow \
+              ingestion-service"
 
 for SERVICE in $SERVICE_LIST;
 do

charts/osdu-common/templates/appgw-ingress.yaml

@@ -99,3 +99,7 @@ spec:
               serviceName: seismic-store-service
               servicePort: 80
             path: /seistore-svc/api/v3/*
+          - backend:
+              serviceName: ingestion-service
+              servicePort: 80
+            path: /api/ingestion/*
\ No newline at end of file

docs/code-mirroring.md

@@ -27,6 +27,7 @@ Empty repositories need to be created that will be used by a pipeline to mirror
 | ingestion-workflow        | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git |
 | seismic-store-service     | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git |
 | wellbore-domain-services  | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git |
+| ingestion-service         | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git |
 
 ```bash
 export ADO_ORGANIZATION=<organization_name>
@@ -55,7 +56,8 @@ SERVICE_LIST="infra-azure-provisioning \
               schema-service \
               ingestion-workflow \
               seismic-store-service \
-              wellbore-domain-services"
+              wellbore-domain-services \
+              ingestion-service"
 
 
 for SERVICE in $SERVICE_LIST;
@@ -95,6 +97,7 @@ Variable Group Name:  `Mirror Variables`
 | INGESTION_WORKFLOW_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-workflow |
 | SEISMIC_STORE_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/seismic-store-service |
 | WELLBORE_DOMAIN_SERVICSE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/wellbore-domain-services |
+| INGESTION_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-service |
 | ACCESS_TOKEN | <your_personal_access_token> |
 
 
@@ -129,6 +132,7 @@ az pipelines variable-group create \
   INGESTION_WORKFLOW_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-workflow \
   SEISMIC_STORE_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/seismic-store-service \
   WELLBORE_DOMAIN_SERVICSE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/wellbore-domain-services \
+  INGESTION_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-service \
   ACCESS_TOKEN=$ACCESS_TOKEN \
   -ojson
 ```
@@ -324,6 +328,13 @@ jobs:
         destinationGitRepositoryUri: '$(WELLBORE_DOMAIN_SERVICSE_REPO)'
         destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
 
+    - task: swellaby.mirror-git-repository.mirror-git-repository-vsts-task.mirror-git-repository-vsts-task@1
+      displayName: 'ingestion-service'
+      inputs:
+        sourceGitRepositoryUri: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git'
+        destinationGitRepositoryUri: '$(INGESTION_SERVICE_REPO)'
+        destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
+
 
 EOF
 

docs/service-automation.md

@@ -662,6 +662,29 @@ az pipelines variable-group create \
   -ojson
 ```
 
+__Setup and Configure the ADO Library `Azure Service Release - ingestion-service`__
+
+This variable group is the service specific variables necessary for testing and deploying the `ingestion` service.
+
+| Variable | Value |
+|----------|-------|
+| MAVEN_DEPLOY_POM_FILE_PATH     | `drop/provider/ingest-azure` |
+
+No Test Path is needed since the service has python tests
+
+```bash
+az pipelines variable-group create \
+  --name "Azure Service Release - ingestion-service" \
+  --authorize true \
+  --variables \
+  MAVEN_DEPLOY_POM_FILE_PATH="drop/provider/ingest-azure" 
+  MAVEN_INTEGRATION_TEST_OPTIONS=`-DargLine=""` \
+  MAVEN_INTEGRATION_TEST_POM_FILE_PATH="drop/deploy/testing/ingest-test-azurepom.xml" \
+  SERVICE_RESOURCE_NAME='$(AZURE_INGESTION_SERVICE_NAME)' \
+  -ojson
+```
+
+
 __Create the Chart Pipelines__
 
 Create the pipelines and run things in this exact order.
@@ -1058,4 +1081,20 @@ az pipelines create \
   --repository-type tfsgit  \
   --yaml-path /devops/azure/pipeline.yml  \
   -ojson
+```
+
+21. Add a Pipeline for __ingestion-service__  to deploy the Ingestion Service.
+
+    _Repo:_ `ingestion-service`
+    _Path:_ `/devops/azure/pipeline.yml`
+    _Validate:_ https://<your_dns_name>/api/ingestion/docs is alive.
+
+```bash
+az pipelines create \
+  --name 'ingestion-service'  \
+  --repository ingestion-service  \
+  --branch master  \
+  --repository-type tfsgit  \
+  --yaml-path /devops/azure/pipeline.yml  \
+  -ojson
 ```
\ No newline at end of file

docs/service-data.md

+# Load Service Data
+
+## Service Schema Loading
+
+Schema Service has standard shared schemas that have to be loaded.
+
+
+```bash
+# Setup Variables
+UNIQUE="<your_osdu_unique>"         # ie: demo
+AZURE_DNS_NAME="<your_osdu_fqdn>"   # ie: osdu-$UNIQUE.contoso.com
+DATA_PARTITION="<your_partition>"   # ie:opendes
+
+# This logs your local Azure CLI in using the configured service principal.
+az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
+
+GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
+ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
+
+cat > .env << EOF
+DATA_PARTITION=$DATA_PARTITION
+AZURE_TENANT_ID=$ARM_TENANT_ID
+AZURE_DNS_NAME=$AZURE_DNS_NAME
+AZURE_AD_APP_RESOURCE_ID=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/aad-client-id --query value -otsv)
+AZURE_CLIENT_ID=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/app-dev-sp-username --query value -otsv)
+AZURE_CLIENT_SECRET=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/app-dev-sp-password --query value -otsv)
+EOF
+
+# Execute container to load the schema's
+docker run --env-file .env msosdu.azurecr.io/osdu-azure-core-load:latest
+```

tools/backup/update_backup_policies.sh

+#!/bin/bash
+
+# Config values for Service Account Data Protection Policies.
+ENABLE_DELETE_RETENTION="true"
+DELETE_RETENTION_DAYS=29
+ENABLE_VERSIONING="true"
+ENABLE_CHANGE_FEED="true"
+ENABLE_RESTORE_POLICY="true"
+RESTORE_DAYS=28
+
+# Config values for CosmosDB Account Backup Policies.
+BACKUP_INTERVAL_IN_MINUTES=480
+BACKUP_RETENTION_IN_HOURS=672
+
+# Literals
+RESOURCETYPE_STORAGE_ACCOUNT="Microsoft.Storage/storageAccounts"
+RESOURCETYPE_COSMOSDB_ACCOUNT="Microsoft.DocumentDb/databaseAccounts"
+QUERY_FOR_NAME='[].name'
+
+# arguments (message)
+function log() {
+  echo >&2 "[update_backup_policies.sh] $1"
+}
+
+function configureDataProtectionPoliciesForStorageAccounts() {
+    log "function:start: ${FUNCNAME}"
+	log "Setting Data Protection policies for all Storage Accounts in the Resource Group: ${resourceGroup}."
+	log "following properties would be updated:"
+	log "DELETE_RETENTION_DAYS: ${DELETE_RETENTION_DAYS}"
+	log "RESTORE_DAYS: ${RESTORE_DAYS}"
+	
+	local resourceGroup=$1
+	
+	storageAccounts=$(az resource list \
+	                    --resource-group "${resourceGroup}" \
+	                    --resource-type "${RESOURCETYPE_STORAGE_ACCOUNT}" \
+	                    --query "${QUERY_FOR_NAME}" \
+	                    --output tsv)
+
+	for storageAccount in $storageAccounts ;
+		do
+			storageAccount=$(echo "${storageAccount}" | sed -r 's/\/r//g')
+			log "Setting backup policies for Storage Account: ${storageAccount}."
+			az storage account blob-service-properties update \
+        --resource-group "${resourceGroup}" \
+        --account-name "${storageAccount}" \
+        --enable-delete-retention "${ENABLE_DELETE_RETENTION}" \
+        --delete-retention-days "${DELETE_RETENTION_DAYS}" \
+        --enable-versioning "${ENABLE_VERSIONING}" \
+        --enable-change-feed "${ENABLE_CHANGE_FEED}" \
+        --enable-restore-policy "${ENABLE_RESTORE_POLICY}" \
+        --restore-days "${RESTORE_DAYS}";
+	  done;
+	log "function:end: ${FUNCNAME}"
+}
+
+function configureBackupPoliciesForCosmosDbAccounts() {
+	log "function:start: ${FUNCNAME}"
+	log "Setting backup policies all CosmosDB Accounts in Resource Group: ${resourceGroup}."
+	log "following properties would be updated:"
+	log "BACKUP_INTERVAL_IN_MINUTES: ${BACKUP_INTERVAL_IN_MINUTES}"
+	log "BACKUP_RETENTION_IN_HOURS: ${BACKUP_RETENTION_IN_HOURS}"	
+
+	local resourceGroup=$1
+	
+	cosmosdbAccounts=$(az resource list \
+	                     --resource-group "${resourceGroup}" \
+	                     --resource-type "${RESOURCETYPE_COSMOSDB_ACCOUNT}" \
+	                     --query "${QUERY_FOR_NAME}" \
+	                     --output tsv)
+	
+	for cosmosDbAccount in $cosmosdbAccounts ;
+		do
+		  cosmosDbAccount=$(echo "${cosmosDbAccount}" | sed -r 's/\/r//g')
+			log "Setting backup policies for CosmosDB Account: ${cosmosDbAccount}."
+			az cosmosdb update \
+				--name "${cosmosDbAccount}"\
+				--resource-group "${resourceGroup}"\
+				--backup-interval "${BACKUP_INTERVAL_IN_MINUTES}" \
+				--backup-retention "${BACKUP_RETENTION_IN_HOURS}" ;
+	  done;
+	log "function:end: ${FUNCNAME}"  
+}
+
+main() {
+	log "function:start: ${FUNCNAME}"
+	local resourceGroup=$1
+	local help=$2
+
+	if [ "$help" == "true" ]; then
+        echo "
+			  Use -r options to specify Resource Group, for which back up is to be configured.
+			  Use -h true option for help
+			  "
+        exit 0
+    fi
+
+	configureDataProtectionPoliciesForStorageAccounts "${resourceGroup}"
+	configureBackupPoliciesForCosmosDbAccounts "${resourceGroup}"
+	log "function:end: ${FUNCNAME}"
+}
+
+# Input Management
+resourceGroup=""
+help="false"
+while getopts ":r::h::" opt; do
+    case $opt in
+        r)
+          resourceGroup=$OPTARG
+          ;;
+        h)
+          help="true"
+          ;;
+		    \?)
+          echo "Invalid option: -$OPTARG"
+          echo "Use -h true option for help"
+          exit 1
+          ;;
+        :)
+          echo "Option -$OPTARG requires an argument."
+          echo "Use -h true option for help"
+          exit 1
+          ;;
+    esac
+done
+
+main "$resourceGroup" "$help"
\ No newline at end of file

tools/variables/ingestion-service.sh

+#!/usr/bin/env bash
+#
+#  Purpose: Create the Developer Environment Variables.
+#  Usage:
+#    ingestion-service.sh
+
+###############################
+## ARGUMENT INPUT            ##
+###############################
+usage() { echo "Usage: DNS_HOST=<your_host> INVALID_JWT=<your_token> ingestion-service.sh " 1>&2; exit 1; }
+
+SERVICE="ingestion-service"
+
+if [ -z $UNIQUE ]; then
+  tput setaf 1; echo 'ERROR: UNIQUE not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $DNS_HOST ]; then
+  tput setaf 1; echo 'ERROR: DNS_HOST not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $COMMON_VAULT ]; then
+  tput setaf 1; echo 'ERROR: COMMON_VAULT not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $INVALID_JWT ]; then
+  tput setaf 1; echo 'ERROR: INVALID_JWT not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -f ./settings_common.env ]; then
+  source ./settings_common.env;
+else
+  tput setaf 1; echo 'ERROR: common.env not found' ; tput sgr0
+fi
+
+if [ -f ./settings_environment.env ]; then
+  source ./settings_environment.env;
+else
+  tput setaf 1; echo 'ERROR: environment.env not found' ; tput sgr0
+fi
+
+if [ ! -d $UNIQUE ]; then mkdir $UNIQUE; fi
+
+
+# ------------------------------------------------------------------------------------------------------
+# LocalHost Run Settings
+# ------------------------------------------------------------------------------------------------------
+ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
+azure_istioauth_enabled="true"
+
+# ------------------------------------------------------------------------------------------------------
+# Integration Test Settings
+# ------------------------------------------------------------------------------------------------------
+INTEGRATION_TESTER="${ENV_PRINCIPAL_ID}"
+TESTER_SERVICEPRINCIPAL_SECRET="${ENV_PRINCIPAL_SECRET}"
+AZURE_TENANT_ID="${TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID="${ENV_APP_ID}"
+BASE_URL=/api/crs/converter/v2
+VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
+client_id="${ENV_PRINCIPAL_ID}"
+MY_TENANT="${OSDU_TENANT}"
+TIME_ZONE="UTC+0"
+
+cat > ${UNIQUE}/${SERVICE}.envrc <<LOCALENV
+# ------------------------------------------------------------------------------------------------------
+# Common Settings
+# ------------------------------------------------------------------------------------------------------
+export OSDU_TENANT=$OSDU_TENANT
+export OSDU_TENANT2=$OSDU_TENANT2
+export OSDU_TENANT3=$OSDU_TENANT3
+export COMPANY_DOMAIN=$COMPANY_DOMAIN
+export COSMOS_DB_NAME=$COSMOS_DB_NAME
+export LEGAL_SERVICE_BUS_TOPIC=$LEGAL_SERVICE_BUS_TOPIC
+export RECORD_SERVICE_BUS_TOPIC=$RECORD_SERVICE_BUS_TOPIC
+export LEGAL_STORAGE_CONTAINER=$LEGAL_STORAGE_CONTAINER
+export TENANT_ID=$TENANT_ID
+export INVALID_JWT=$INVALID_JWT
+
+export NO_ACCESS_ID=$NO_ACCESS_ID
+export NO_ACCESS_SECRET=$NO_ACCESS_SECRET
+export OTHER_APP_ID=$OTHER_APP_ID
+export OTHER_APP_OID=$OTHER_APP_OID
+
+export AD_USER_EMAIL=$AD_USER_EMAIL
+export AD_USER_OID=$AD_USER_OID
+export AD_GUEST_EMAIL=$AD_GUEST_EMAIL
+export AD_GUEST_OID=$AD_GUEST_OID
+
+# ------------------------------------------------------------------------------------------------------
+# Environment Settings
+# ------------------------------------------------------------------------------------------------------
+export ENV_SUBSCRIPTION_NAME=$ENV_SUBSCRIPTION_NAME
+export ENV_APP_ID=$ENV_APP_ID
+export ENV_PRINCIPAL_ID=$ENV_PRINCIPAL_ID
+export ENV_PRINCIPAL_SECRET=$ENV_PRINCIPAL_SECRET
+export ENV_APPINSIGHTS_KEY=$ENV_APPINSIGHTS_KEY
+export ENV_REGISTRY=$ENV_REGISTRY
+export ENV_STORAGE=$ENV_STORAGE
+export ENV_STORAGE_KEY=$ENV_STORAGE_KEY
+export ENV_STORAGE_CONNECTION=$ENV_STORAGE_CONNECTION
+export ENV_COSMOSDB_HOST=$ENV_COSMOSDB_HOST
+export ENV_COSMOSDB_KEY=$ENV_COSMOSDB_KEY
+export ENV_SERVICEBUS_NAMESPACE=$ENV_SERVICEBUS_NAMESPACE
+export ENV_SERVICEBUS_CONNECTION=$ENV_SERVICEBUS_CONNECTION
+export ENV_KEYVAULT=$ENV_KEYVAULT
+export ENV_HOST=$ENV_HOST
+export ENV_REGION=$ENV_REGION
+export ENV_ELASTIC_HOST=$ENV_ELASTIC_HOST
+export ENV_ELASTIC_PORT=$ENV_ELASTIC_PORT
+export ENV_ELASTIC_USERNAME=$ENV_ELASTIC_USERNAME
+export ENV_ELASTIC_PASSWORD=$ENV_ELASTIC_PASSWORD
+
+
+# ------------------------------------------------------------------------------------------------------
+# LocalHost Run Settings
+# ------------------------------------------------------------------------------------------------------
+export ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
+export azure_istioauth_enabled="true"
+
+# ------------------------------------------------------------------------------------------------------
+# Integration Test Settings
+# ------------------------------------------------------------------------------------------------------
+export INTEGRATION_TESTER="${INTEGRATION_TESTER}"
+export TESTER_SERVICEPRINCIPAL_SECRET="${TESTER_SERVICEPRINCIPAL_SECRET}"
+export AZURE_TENANT_ID="${AZURE_TENANT_ID}"
+export AZURE_AD_APP_RESOURCE_ID="${AZURE_AD_APP_RESOURCE_ID}"
+export BASE_URL=/api/crs/converter/v2
+export VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
+export client_id="${ENV_PRINCIPAL_ID}"
+export MY_TENANT="${OSDU_TENANT}"
+export TIME_ZONE="${TIME_ZONE}"
+LOCALENV
+
+
+cat > ${UNIQUE}/${SERVICE}_local.yaml <<LOCALRUN
+ENTITLEMENTS_URL: "${ENTITLEMENTS_URL}
+azure_istioauth_enabled: "${azure_istioauth_enabled}"
+LOCALRUN
+
+
+cat > ${UNIQUE}/${SERVICE}_local_test.yaml <<LOCALTEST
+INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
+TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
+AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
+BASE_URL: "${BASE_URL}"
+VIRTUAL_SERVICE_HOST_NAME: "${VIRTUAL_SERVICE_HOST_NAME}"
+client_id: "${client_id}"
+MY_TENANT: "${MY_TENANT}"
+TIME_ZONE: "${TIME_ZONE}"
+LOCALTEST
+
+
+cat > ${UNIQUE}/${SERVICE}_test.yaml <<DEVTEST
+INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
+TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
+AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
+BASE_URL: "${BASE_URL}"
+VIRTUAL_SERVICE_HOST_NAME: "${ENV_HOST}"
+client_id: "${client_id}"
+MY_TENANT: "${MY_TENANT}"
+TIME_ZONE: "${TIME_ZONE}"
+DEVTEST