Commit e8974b8c authored by Sumra Zafar's avatar Sumra Zafar Committed by Jason
Browse files

Adding ingestion service

parent 8967939a
......@@ -331,6 +331,7 @@ git clone https://community.opengroup.org/osdu/platform/system/schema-service.gi
git clonehttps://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git $SRC_DIR/ingestion-workflow
git clone https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git $SRC_DIR/seismic-store-service
git clone https://community.opengroup.org:osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git $SRC_DIR/wellbore-domain-services
git clone https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git $SRC_DIR/ingestion-service
```
__Additional Manual Steps__
......@@ -433,7 +434,8 @@ SERVICE_LIST="infra-azure-provisioning \
register \
notification \
schema-service \
ingestion-workflow"
ingestion-workflow \
ingestion-service"
for SERVICE in $SERVICE_LIST;
do
......
......@@ -99,3 +99,7 @@ spec:
serviceName: seismic-store-service
servicePort: 80
path: /seistore-svc/api/v3/*
- backend:
serviceName: ingestion-service
servicePort: 80
path: /api/ingestion/*
\ No newline at end of file
......@@ -27,6 +27,7 @@ Empty repositories need to be created that will be used by a pipeline to mirror
| ingestion-workflow | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git |
| seismic-store-service | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git |
| wellbore-domain-services | https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services.git |
| ingestion-service | https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git |
```bash
export ADO_ORGANIZATION=<organization_name>
......@@ -55,7 +56,8 @@ SERVICE_LIST="infra-azure-provisioning \
schema-service \
ingestion-workflow \
seismic-store-service \
wellbore-domain-services"
wellbore-domain-services \
ingestion-service"
for SERVICE in $SERVICE_LIST;
......@@ -95,6 +97,7 @@ Variable Group Name: `Mirror Variables`
| INGESTION_WORKFLOW_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-workflow |
| SEISMIC_STORE_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/seismic-store-service |
| WELLBORE_DOMAIN_SERVICSE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/wellbore-domain-services |
| INGESTION_SERVICE_REPO | https://dev.azure.com/osdu-demo/osdu/_git/ingestion-service |
| ACCESS_TOKEN | <your_personal_access_token> |
......@@ -129,6 +132,7 @@ az pipelines variable-group create \
INGESTION_WORKFLOW_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-workflow \
SEISMIC_STORE_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/seismic-store-service \
WELLBORE_DOMAIN_SERVICSE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/wellbore-domain-services \
INGESTION_SERVICE_REPO=https://dev.azure.com/${ADO_ORGANIZATION}/$ADO_PROJECT/_git/ingestion-service \
ACCESS_TOKEN=$ACCESS_TOKEN \
-ojson
```
......@@ -324,6 +328,13 @@ jobs:
destinationGitRepositoryUri: '$(WELLBORE_DOMAIN_SERVICSE_REPO)'
destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
- task: swellaby.mirror-git-repository.mirror-git-repository-vsts-task.mirror-git-repository-vsts-task@1
displayName: 'ingestion-service'
inputs:
sourceGitRepositoryUri: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-service.git'
destinationGitRepositoryUri: '$(INGESTION_SERVICE_REPO)'
destinationGitRepositoryPersonalAccessToken: $(ACCESS_TOKEN)
EOF
......
......@@ -662,6 +662,29 @@ az pipelines variable-group create \
-ojson
```
__Setup and Configure the ADO Library `Azure Service Release - ingestion-service`__
This variable group is the service specific variables necessary for testing and deploying the `ingestion` service.
| Variable | Value |
|----------|-------|
| MAVEN_DEPLOY_POM_FILE_PATH | `drop/provider/ingest-azure` |
No Test Path is needed since the service has python tests
```bash
az pipelines variable-group create \
--name "Azure Service Release - ingestion-service" \
--authorize true \
--variables \
MAVEN_DEPLOY_POM_FILE_PATH="drop/provider/ingest-azure"
MAVEN_INTEGRATION_TEST_OPTIONS=`-DargLine=""` \
MAVEN_INTEGRATION_TEST_POM_FILE_PATH="drop/deploy/testing/ingest-test-azurepom.xml" \
SERVICE_RESOURCE_NAME='$(AZURE_INGESTION_SERVICE_NAME)' \
-ojson
```
__Create the Chart Pipelines__
Create the pipelines and run things in this exact order.
......@@ -1058,4 +1081,20 @@ az pipelines create \
--repository-type tfsgit \
--yaml-path /devops/azure/pipeline.yml \
-ojson
```
21. Add a Pipeline for __ingestion-service__ to deploy the Ingestion Service.
_Repo:_ `ingestion-service`
_Path:_ `/devops/azure/pipeline.yml`
_Validate:_ https://<your_dns_name>/api/ingestion/docs is alive.
```bash
az pipelines create \
--name 'ingestion-service' \
--repository ingestion-service \
--branch master \
--repository-type tfsgit \
--yaml-path /devops/azure/pipeline.yml \
-ojson
```
\ No newline at end of file
#!/usr/bin/env bash
#
# Purpose: Create the Developer Environment Variables.
# Usage:
# ingestion-service.sh
###############################
## ARGUMENT INPUT ##
###############################
usage() { echo "Usage: DNS_HOST=<your_host> INVALID_JWT=<your_token> ingestion-service.sh " 1>&2; exit 1; }
SERVICE="ingestion-service"
if [ -z $UNIQUE ]; then
tput setaf 1; echo 'ERROR: UNIQUE not provided' ; tput sgr0
usage;
fi
if [ -z $DNS_HOST ]; then
tput setaf 1; echo 'ERROR: DNS_HOST not provided' ; tput sgr0
usage;
fi
if [ -z $COMMON_VAULT ]; then
tput setaf 1; echo 'ERROR: COMMON_VAULT not provided' ; tput sgr0
usage;
fi
if [ -z $INVALID_JWT ]; then
tput setaf 1; echo 'ERROR: INVALID_JWT not provided' ; tput sgr0
usage;
fi
if [ -f ./settings_common.env ]; then
source ./settings_common.env;
else
tput setaf 1; echo 'ERROR: common.env not found' ; tput sgr0
fi
if [ -f ./settings_environment.env ]; then
source ./settings_environment.env;
else
tput setaf 1; echo 'ERROR: environment.env not found' ; tput sgr0
fi
if [ ! -d $UNIQUE ]; then mkdir $UNIQUE; fi
# ------------------------------------------------------------------------------------------------------
# LocalHost Run Settings
# ------------------------------------------------------------------------------------------------------
ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
azure_istioauth_enabled="true"
# ------------------------------------------------------------------------------------------------------
# Integration Test Settings
# ------------------------------------------------------------------------------------------------------
INTEGRATION_TESTER="${ENV_PRINCIPAL_ID}"
TESTER_SERVICEPRINCIPAL_SECRET="${ENV_PRINCIPAL_SECRET}"
AZURE_TENANT_ID="${TENANT_ID}"
AZURE_AD_APP_RESOURCE_ID="${ENV_APP_ID}"
BASE_URL=/api/crs/converter/v2
VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
client_id="${ENV_PRINCIPAL_ID}"
MY_TENANT="${OSDU_TENANT}"
TIME_ZONE="UTC+0"
cat > ${UNIQUE}/${SERVICE}.envrc <<LOCALENV
# ------------------------------------------------------------------------------------------------------
# Common Settings
# ------------------------------------------------------------------------------------------------------
export OSDU_TENANT=$OSDU_TENANT
export OSDU_TENANT2=$OSDU_TENANT2
export OSDU_TENANT3=$OSDU_TENANT3
export COMPANY_DOMAIN=$COMPANY_DOMAIN
export COSMOS_DB_NAME=$COSMOS_DB_NAME
export LEGAL_SERVICE_BUS_TOPIC=$LEGAL_SERVICE_BUS_TOPIC
export RECORD_SERVICE_BUS_TOPIC=$RECORD_SERVICE_BUS_TOPIC
export LEGAL_STORAGE_CONTAINER=$LEGAL_STORAGE_CONTAINER
export TENANT_ID=$TENANT_ID
export INVALID_JWT=$INVALID_JWT
export NO_ACCESS_ID=$NO_ACCESS_ID
export NO_ACCESS_SECRET=$NO_ACCESS_SECRET
export OTHER_APP_ID=$OTHER_APP_ID
export OTHER_APP_OID=$OTHER_APP_OID
export AD_USER_EMAIL=$AD_USER_EMAIL
export AD_USER_OID=$AD_USER_OID
export AD_GUEST_EMAIL=$AD_GUEST_EMAIL
export AD_GUEST_OID=$AD_GUEST_OID
# ------------------------------------------------------------------------------------------------------
# Environment Settings
# ------------------------------------------------------------------------------------------------------
export ENV_SUBSCRIPTION_NAME=$ENV_SUBSCRIPTION_NAME
export ENV_APP_ID=$ENV_APP_ID
export ENV_PRINCIPAL_ID=$ENV_PRINCIPAL_ID
export ENV_PRINCIPAL_SECRET=$ENV_PRINCIPAL_SECRET
export ENV_APPINSIGHTS_KEY=$ENV_APPINSIGHTS_KEY
export ENV_REGISTRY=$ENV_REGISTRY
export ENV_STORAGE=$ENV_STORAGE
export ENV_STORAGE_KEY=$ENV_STORAGE_KEY
export ENV_STORAGE_CONNECTION=$ENV_STORAGE_CONNECTION
export ENV_COSMOSDB_HOST=$ENV_COSMOSDB_HOST
export ENV_COSMOSDB_KEY=$ENV_COSMOSDB_KEY
export ENV_SERVICEBUS_NAMESPACE=$ENV_SERVICEBUS_NAMESPACE
export ENV_SERVICEBUS_CONNECTION=$ENV_SERVICEBUS_CONNECTION
export ENV_KEYVAULT=$ENV_KEYVAULT
export ENV_HOST=$ENV_HOST
export ENV_REGION=$ENV_REGION
export ENV_ELASTIC_HOST=$ENV_ELASTIC_HOST
export ENV_ELASTIC_PORT=$ENV_ELASTIC_PORT
export ENV_ELASTIC_USERNAME=$ENV_ELASTIC_USERNAME
export ENV_ELASTIC_PASSWORD=$ENV_ELASTIC_PASSWORD
# ------------------------------------------------------------------------------------------------------
# LocalHost Run Settings
# ------------------------------------------------------------------------------------------------------
export ENTITLEMENTS_URL="https://${ENV_HOST}/entitlements/v1"
export azure_istioauth_enabled="true"
# ------------------------------------------------------------------------------------------------------
# Integration Test Settings
# ------------------------------------------------------------------------------------------------------
export INTEGRATION_TESTER="${INTEGRATION_TESTER}"
export TESTER_SERVICEPRINCIPAL_SECRET="${TESTER_SERVICEPRINCIPAL_SECRET}"
export AZURE_TENANT_ID="${AZURE_TENANT_ID}"
export AZURE_AD_APP_RESOURCE_ID="${AZURE_AD_APP_RESOURCE_ID}"
export BASE_URL=/api/crs/converter/v2
export VIRTUAL_SERVICE_HOST_NAME="localhost:8080"
export client_id="${ENV_PRINCIPAL_ID}"
export MY_TENANT="${OSDU_TENANT}"
export TIME_ZONE="${TIME_ZONE}"
LOCALENV
cat > ${UNIQUE}/${SERVICE}_local.yaml <<LOCALRUN
ENTITLEMENTS_URL: "${ENTITLEMENTS_URL}
azure_istioauth_enabled: "${azure_istioauth_enabled}"
LOCALRUN
cat > ${UNIQUE}/${SERVICE}_local_test.yaml <<LOCALTEST
INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
BASE_URL: "${BASE_URL}"
VIRTUAL_SERVICE_HOST_NAME: "${VIRTUAL_SERVICE_HOST_NAME}"
client_id: "${client_id}"
MY_TENANT: "${MY_TENANT}"
TIME_ZONE: "${TIME_ZONE}"
LOCALTEST
cat > ${UNIQUE}/${SERVICE}_test.yaml <<DEVTEST
INTEGRATION_TESTER: "${INTEGRATION_TESTER}"
TESTER_SERVICEPRINCIPAL_SECRET: "${TESTER_SERVICEPRINCIPAL_SECRET}"
AZURE_TENANT_ID: "${AZURE_TENANT_ID}"
AZURE_AD_APP_RESOURCE_ID: "${AZURE_AD_APP_RESOURCE_ID}"
BASE_URL: "${BASE_URL}"
VIRTUAL_SERVICE_HOST_NAME: "${ENV_HOST}"
client_id: "${client_id}"
MY_TENANT: "${MY_TENANT}"
TIME_ZONE: "${TIME_ZONE}"
DEVTEST
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment