Adding ingestion service

charts/README.md

@@ -331,6 +331,7 @@ git clone https://community.opengroup.org/osdu/platform/system/schema-service.gi
 git clonehttps://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git $SRC_DIR/ingestion-workflow
 git clone https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git $SRC_DIR/seismic-store-service
 git clone https://community.opengroup.org:osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git $SRC_DIR/wellbore-domain-services
+git clone https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git $SRC_DIR/ingestion-service
 ```
 
 __Additional Manual Steps__
@@ -433,7 +434,8 @@ SERVICE_LIST="infra-azure-provisioning \
               register \
               notification \
               schema-service \
-              ingestion-workflow"
+              ingestion-workflow \
+              ingestion-service"
 
 for SERVICE in $SERVICE_LIST;
 do

charts/osdu-common/templates/appgw-ingress.yaml

@@ -99,3 +99,7 @@ spec:
               serviceName: seismic-store-service
               servicePort: 80
             path: /seistore-svc/api/v3/*
+          - backend:
+              serviceName: ingestion-service
+              servicePort: 80
+            path: /api/ingestion/*
\ No newline at end of file

docs/code-mirroring.md

@@ -27,6 +27,7 @@ Empty repositories need to be created that will be used by a pipeline to mirror
 | ingestion-workflow        | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git |
 | seismic-store-service     | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git |
 | wellbore-domain-services  | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git |
+| ingestion-service         | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git |
 
 ```bash
 export ADO_ORGANIZATION=<organization_name>
@@ -55,7 +56,8 @@ SERVICE_LIST="infra-azure-provisioning \
               schema-service \
               ingestion-workflow \
               seismic-store-service \
-              wellbore-domain-services"
+              wellbore-domain-services \
+              ingestion-service"
 
 
 for SERVICE in $SERVICE_LIST;
@@ -95,6 +97,7 @@ Variable Group Name:  `Mirror Variables`
 | INGESTION_WORKFLOW_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-workflow |
 | SEISMIC_STORE_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/seismic-store-service |
 | WELLBORE_DOMAIN_SERVICSE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/wellbore-domain-services |
+| INGESTION_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-service |
 | ACCESS_TOKEN | <your_personal_access_token> |
 
 
@@ -129,6 +132,7 @@ az pipelines variable-group create \
   INGESTION_WORKFLOW_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-workflow \
   SEISMIC_STORE_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/seismic-store-service \
   WELLBORE_DOMAIN_SERVICSE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/wellbore-domain-services \
+  INGESTION_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-service \
   ACCESS_TOKEN=$ACCESS_TOKEN \
   -ojson
 ```
@@ -324,6 +328,13 @@ jobs:
         destinationGitRepositoryUri: '$(WELLBORE_DOMAIN_SERVICSE_REPO)'
         destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
 
+    - task: swellaby.mirror-git-repository.mirror-git-repository-vsts-task.mirror-git-repository-vsts-task@1
+      displayName: 'ingestion-service'
+      inputs:
+        sourceGitRepositoryUri: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git'
+        destinationGitRepositoryUri: '$(INGESTION_SERVICE_REPO)'
+        destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
+
 
 EOF
 

docs/service-automation.md

@@ -662,6 +662,29 @@ az pipelines variable-group create \
   -ojson
 ```
 
+__Setup and Configure the ADO Library `Azure Service Release - ingestion-service`__
+
+This variable group is the service specific variables necessary for testing and deploying the `ingestion` service.
+
+| Variable | Value |
+|----------|-------|
+| MAVEN_DEPLOY_POM_FILE_PATH     | `drop/provider/ingest-azure` |
+
+No Test Path is needed since the service has python tests
+
+```bash
+az pipelines variable-group create \
+  --name "Azure Service Release - ingestion-service" \
+  --authorize true \
+  --variables \
+  MAVEN_DEPLOY_POM_FILE_PATH="drop/provider/ingest-azure" 
+  MAVEN_INTEGRATION_TEST_OPTIONS=`-DargLine=""` \
+  MAVEN_INTEGRATION_TEST_POM_FILE_PATH="drop/deploy/testing/ingest-test-azurepom.xml" \
+  SERVICE_RESOURCE_NAME='$(AZURE_INGESTION_SERVICE_NAME)' \
+  -ojson
+```
+
+
 __Create the Chart Pipelines__
 
 Create the pipelines and run things in this exact order.
@@ -1058,4 +1081,20 @@ az pipelines create \
   --repository-type tfsgit  \
   --yaml-path /devops/azure/pipeline.yml  \
   -ojson
+```
+
+21. Add a Pipeline for __ingestion-service__  to deploy the Ingestion Service.
+
+    _Repo:_ `ingestion-service`
+    _Path:_ `/devops/azure/pipeline.yml`
+    _Validate:_ https://<your_dns_name>/api/ingestion/docs is alive.
+
+```bash
+az pipelines create \
+  --name 'ingestion-service'  \
+  --repository ingestion-service  \
+  --branch master  \
+  --repository-type tfsgit  \
+  --yaml-path /devops/azure/pipeline.yml  \
+  -ojson
 ```
\ No newline at end of file

tools/variables/ingestion-service.sh

+#!/usr/bin/env bash
+#
+#  Purpose: Create the Developer Environment Variables.
+#  Usage:
+#    ingestion-service.sh
+
+###############################
+## ARGUMENT INPUT            ##
+###############################
+usage() { echo "Usage: DNS_HOST=<your_host> INVALID_JWT=<your_token> ingestion-service.sh " 1>&2; exit 1; }
+
+SERVICE="ingestion-service"
+
+if [ -z $UNIQUE ]; then
+  tput setaf 1; echo 'ERROR: UNIQUE not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $DNS_HOST ]; then
+  tput setaf 1; echo 'ERROR: DNS_HOST not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $COMMON_VAULT ]; then
+  tput setaf 1; echo 'ERROR: COMMON_VAULT not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -z $INVALID_JWT ]; then
+  tput setaf 1; echo 'ERROR: INVALID_JWT not provided' ; tput sgr0
+  usage;
+fi
+
+if [ -f ./settings_common.env ]; then
+  source ./settings_common.env;
+else
+  tput setaf 1; echo 'ERROR: common.env not found' ; tput sgr0
+fi
+
+if [ -f ./settings_environment.env ]; then
+  source ./settings_environment.env;
+else
+  tput setaf 1; echo 'ERROR: environment.env not found' ; tput sgr0
+fi
+
+if [ ! -d $UNIQUE ]; then mkdir $UNIQUE; fi
+
+
+# ------------------------------------------------------------------------------------------------------
+# LocalHost Run Settings
+# ------------------------------------------------------------------------------------------------------
+ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
+azure_istioauth_enabled="true"
+
+# ------------------------------------------------------------------------------------------------------
+# Integration Test Settings
+# ------------------------------------------------------------------------------------------------------
+INTEGRATION_TESTER="${ENV_PRINCIPAL_ID}"
+TESTER_SERVICEPRINCIPAL_SECRET="${ENV_PRINCIPAL_SECRET}"
+AZURE_TENANT_ID="${TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID="${ENV_APP_ID}"
+BASE_URL=/api/crs/converter/v2
+VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
+client_id="${ENV_PRINCIPAL_ID}"
+MY_TENANT="${OSDU_TENANT}"
+TIME_ZONE="UTC+0"
+
+cat > ${UNIQUE}/${SERVICE}.envrc <<LOCALENV
+# ------------------------------------------------------------------------------------------------------
+# Common Settings
+# ------------------------------------------------------------------------------------------------------
+export OSDU_TENANT=$OSDU_TENANT
+export OSDU_TENANT2=$OSDU_TENANT2
+export OSDU_TENANT3=$OSDU_TENANT3
+export COMPANY_DOMAIN=$COMPANY_DOMAIN
+export COSMOS_DB_NAME=$COSMOS_DB_NAME
+export LEGAL_SERVICE_BUS_TOPIC=$LEGAL_SERVICE_BUS_TOPIC
+export RECORD_SERVICE_BUS_TOPIC=$RECORD_SERVICE_BUS_TOPIC
+export LEGAL_STORAGE_CONTAINER=$LEGAL_STORAGE_CONTAINER
+export TENANT_ID=$TENANT_ID
+export INVALID_JWT=$INVALID_JWT
+
+export NO_ACCESS_ID=$NO_ACCESS_ID
+export NO_ACCESS_SECRET=$NO_ACCESS_SECRET
+export OTHER_APP_ID=$OTHER_APP_ID
+export OTHER_APP_OID=$OTHER_APP_OID
+
+export AD_USER_EMAIL=$AD_USER_EMAIL
+export AD_USER_OID=$AD_USER_OID
+export AD_GUEST_EMAIL=$AD_GUEST_EMAIL
+export AD_GUEST_OID=$AD_GUEST_OID
+
+# ------------------------------------------------------------------------------------------------------
+# Environment Settings
+# ------------------------------------------------------------------------------------------------------
+export ENV_SUBSCRIPTION_NAME=$ENV_SUBSCRIPTION_NAME
+export ENV_APP_ID=$ENV_APP_ID
+export ENV_PRINCIPAL_ID=$ENV_PRINCIPAL_ID
+export ENV_PRINCIPAL_SECRET=$ENV_PRINCIPAL_SECRET
+export ENV_APPINSIGHTS_KEY=$ENV_APPINSIGHTS_KEY
+export ENV_REGISTRY=$ENV_REGISTRY
+export ENV_STORAGE=$ENV_STORAGE
+export ENV_STORAGE_KEY=$ENV_STORAGE_KEY
+export ENV_STORAGE_CONNECTION=$ENV_STORAGE_CONNECTION
+export ENV_COSMOSDB_HOST=$ENV_COSMOSDB_HOST
+export ENV_COSMOSDB_KEY=$ENV_COSMOSDB_KEY
+export ENV_SERVICEBUS_NAMESPACE=$ENV_SERVICEBUS_NAMESPACE
+export ENV_SERVICEBUS_CONNECTION=$ENV_SERVICEBUS_CONNECTION
+export ENV_KEYVAULT=$ENV_KEYVAULT
+export ENV_HOST=$ENV_HOST
+export ENV_REGION=$ENV_REGION
+export ENV_ELASTIC_HOST=$ENV_ELASTIC_HOST
+export ENV_ELASTIC_PORT=$ENV_ELASTIC_PORT
+export ENV_ELASTIC_USERNAME=$ENV_ELASTIC_USERNAME
+export ENV_ELASTIC_PASSWORD=$ENV_ELASTIC_PASSWORD
+
+
+# ------------------------------------------------------------------------------------------------------
+# LocalHost Run Settings
+# ------------------------------------------------------------------------------------------------------
+export ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
+export azure_istioauth_enabled="true"
+
+# ------------------------------------------------------------------------------------------------------
+# Integration Test Settings
+# ------------------------------------------------------------------------------------------------------
+export INTEGRATION_TESTER="${INTEGRATION_TESTER}"
+export TESTER_SERVICEPRINCIPAL_SECRET="${TESTER_SERVICEPRINCIPAL_SECRET}"
+export AZURE_TENANT_ID="${AZURE_TENANT_ID}"
+export AZURE_AD_APP_RESOURCE_ID="${AZURE_AD_APP_RESOURCE_ID}"
+export BASE_URL=/api/crs/converter/v2
+export VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
+export client_id="${ENV_PRINCIPAL_ID}"
+export MY_TENANT="${OSDU_TENANT}"
+export TIME_ZONE="${TIME_ZONE}"
+LOCALENV
+
+
+cat > ${UNIQUE}/${SERVICE}_local.yaml <<LOCALRUN
+ENTITLEMENTS_URL: "${ENTITLEMENTS_URL}
+azure_istioauth_enabled: "${azure_istioauth_enabled}"
+LOCALRUN
+
+
+cat > ${UNIQUE}/${SERVICE}_local_test.yaml <<LOCALTEST
+INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
+TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
+AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
+BASE_URL: "${BASE_URL}"
+VIRTUAL_SERVICE_HOST_NAME: "${VIRTUAL_SERVICE_HOST_NAME}"
+client_id: "${client_id}"
+MY_TENANT: "${MY_TENANT}"
+TIME_ZONE: "${TIME_ZONE}"
+LOCALTEST
+
+
+cat > ${UNIQUE}/${SERVICE}_test.yaml <<DEVTEST
+INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
+TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
+AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
+AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
+BASE_URL: "${BASE_URL}"
+VIRTUAL_SERVICE_HOST_NAME: "${ENV_HOST}"
+client_id: "${client_id}"
+MY_TENANT: "${MY_TENANT}"
+TIME_ZONE: "${TIME_ZONE}"
+DEVTEST