Skip to content
Snippets Groups Projects
Commit cbfd04af authored by Dania Kodeih (Microsoft)'s avatar Dania Kodeih (Microsoft)
Browse files

Merge branch 'trusted-azure' into 'master' 

Enable Azure Cloud Provider Builds and Integration Tests for service

See merge request !3
parents 8587f80f 941fef9d
No related branches found
No related tags found
2 merge requests!4Test/update parameter store,!3Enable Azure Cloud Provider Builds and Integration Tests for service
Pipeline #2113 failed
Stage: build
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: attribution
Expand all Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 16
− 3
View file @ cbfd04af
......@@ -2,6 +2,7 @@ variables:
AWS_BUILD_SUBDIR: provider/indexer-aws
AWS_APPLICATION_NAME: os-indexer
AWS_ENVIRONMENT: dev
GCP_BUILD_SUBDIR: provider/indexer-gcp
GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
GCP_APPLICATION_NAME: osdu-indexer
......@@ -12,7 +13,19 @@ variables:
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
AZURE_SERVICE: indexer
AZURE_BUILD_SUBDIR: provider/indexer-azure
AZURE_TEST_SUBDIR: testing/indexer-test-azure
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'temp-service.gitlab-ci.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "build/maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "master"
file: "cloud-providers/azure.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
.mvn/community-maven.settings.xml
+ 58
− 48
View file @ cbfd04af
<settings>
<profiles>
<profile>
<!-- This profile uses the CI-Token to authenticate with the server, and is the default case -->
<id>GitLab-Authenticate-With-Job-Token</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
</properties>
</profile>
<profiles>
<profile>
<!-- This profile uses the CI-Token to authenticate with the server, and is the default case -->
<id>GitLab-Authenticate-With-Job-Token</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
</properties>
</profile>
<profile>
<!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
<profile>
<!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
</properties>
</profile>
</profiles>
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
</properties>
</profile>
</profiles>
<servers>
<server>
<id>community-maven-via-job-token</id>
<configuration>
<httpHeaders>
<property>
<name>Job-Token</name>
<value>${env.CI_JOB_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
<servers>
<server>
<id>community-maven-via-job-token</id>
<configuration>
<httpHeaders>
<property>
<name>Job-Token</name>
<value>${env.CI_JOB_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
<server>
<id>community-maven-via-private-token</id>
<configuration>
<httpHeaders>
<property>
<name>Private-Token</name>
<value>${env.COMMUNITY_MAVEN_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
</servers>
<server>
<id>community-maven-via-private-token</id>
<configuration>
<httpHeaders>
<property>
<name>Private-Token</name>
<value>${env.COMMUNITY_MAVEN_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
<server>
<id>azure-auth</id>
<configuration>
<tenant>${env.AZURE_TENANT_ID}</tenant>
<client>${env.AZURE_CLIENT_ID}</client>
<key>${env.AZURE_CLIENT_SECRET}</key>
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
NOTICE
+ 290
− 11
View file @ cbfd04af
This diff is collapsed.
devops/azure-pipeline.yml 0 → 100644
+ 63
− 0
View file @ cbfd04af
#####################
# README: Defines a template to be used as a starting point for defining a service pipeline
#####################
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- images/
pr:
autoCancel: true
branches:
include:
- '*'
paths:
exclude:
- /**/*.md
- .gitignore
- images/
resources:
repositories:
- repository: osdu-infrastructure
type: git
name: osdu-r2/osdu-infrastructure
variables:
- group: 'Azure Common Secrets'
- group: 'Azure - Common'
- name: serviceName
value: 'indexer'
stages:
- template: devops/service-pipelines/build-stage.yml@osdu-infrastructure
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P indexer-core'
mavenOptions: '-P indexer-azure'
copyFileContents: |
pom.xml
provider/indexer-azure/maven/settings.xml
provider/indexer-azure/pom.xml
provider/indexer-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
- template: devops/service-pipelines/deploy-stages.yml@osdu-infrastructure
parameters:
serviceName: ${{ variables.serviceName }}
testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
providers:
- name: Azure
environments: ['demo']
provider/indexer-azure/README.md
+ 104
− 6
View file @ cbfd04af
......@@ -88,7 +88,7 @@ Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b0
...
```
You will need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`:
You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`:
```bash
$ cat ~/.m2/settings.xml
<?xml version="1.0" encoding="UTF-8"?>
......@@ -123,13 +123,16 @@ $ (cd provider/indexer-azure/ && mvn clean package)
# Note: this assumes that the environment variables for running the service as outlined
# above are already exported in your environment.
$ java -jar $(find provider/indexer-azure/target/ -name *-spring-boot.jar)
# Alternately you can run using the Mavan Task
$ mvn spring-boot:run
```
### Test the application
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests.
> **Note**: the integration tests for `os-indexer-azure` work by validating that records submitted to `os-storage-azure` can eventually be queried by `os-search-azure`. This only works if the messages emitted by `os-storage-azure` can be consumed by `os-indexer-queue-azure`, which will submit the indexing request to `os-indexer-azure`.
> **Note**: the integration tests for `os-indexer-azure` work by validating that records submitted to `os-storage-azure` can eventually be queried by `os-search-azure`. This only works if the messages emitted by `os-storage-azure` can be consumed by `os-indexer-queue-azure`, which will submit the indexing request to `os-indexer-azure`.
>
> In order to make sure that the integration tests are running against your local environment, you will need to make sure that the there is an instance of `os-indexer-queue-azure` that is configured to call your deployment of `os-indexer-azure`, and that this instance of `os-indexer-queue-azure` is the only consumer of the Service Bus topic.
>
......@@ -156,8 +159,103 @@ Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](htt
## Deploying service to Azure
Service deployments into Azure are standardized to make the process the same for all services. The steps to deploy into
Azure can be [found here](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/infrastructure-templates?path=%2Fdocs%2Fosdu%2FSERVICE_DEPLOYMENTS.md&_a=preview)
Service deployments into Azure are standardized to make the process the same for all services if using ADO and are closely related to the infrastructure deployed. The steps to deploy into Azure can be [found here](https://github.com/azure/osdu-infrastructure)
The default ADO pipeline is /devops/azure-pipeline.yml
### Manual Deployment Steps
__Environment Settings__
The following environment variables are necessary to properly deploy a service to an Azure OSDU Environment.
```bash
# Group Level Variables
export AZURE_TENANT_ID=""
export AZURE_SUBSCRIPTION_ID=""
export AZURE_SUBSCRIPTION_NAME=""
export AZURE_PRINCIPAL_ID=""
export AZURE_PRINCIPAL_SECRET=""
export AZURE_APP_ID=""
export AZURE_BASENAME_21=""
export AZURE_BASENAME=""
export AZURE_BASE=""
export AZURE_ELASTIC_HOST=""
export AZURE_ELASTIC_PASSWORD=""
# Pipeline Level Variable
export AZURE_SERVICE="indexer"
export AZURE_BUILD_SUBDIR="provider/indexer-azure"
export AZURE_TEST_SUBDIR="testing/indexer-test-azure"
# Required for Azure Deployment
export AZURE_CLIENT_ID="${AZURE_PRINCIPAL_ID}"
export AZURE_CLIENT_SECRET="${AZURE_PRINCIPAL_SECRET}"
export AZURE_RESOURCE_GROUP="${AZURE_BASENAME}-osdu-r2-app-rg"
export AZURE_APPSERVICE_PLAN="${AZURE_BASENAME}-osdu-r2-sp"
export AZURE_APPSERVICE_NAME="${AZURE_BASENAME_21}-au-${AZURE_SERVICE}"
# Required for Testing
export AZURE_AD_TENANT_ID="$AZURE_TENANT_ID"
export INTEGRATION_TESTER="$AZURE_PRINCIPAL_ID"
export AZURE_TESTER_SERVICEPRINCIPAL_SECRET="$AZURE_PRINCIPAL_SECRET"
export AZURE_AD_APP_RESOURCE_ID="$AZURE_APP_ID"
export aad_client_id="$AZURE_APP_ID"
export STORAGE_HOST="https://{AZURE_BASENAME_21}-au-storage.azurewebsites.net/"
export ELASTIC_HOST="$AZURE_ELASTIC_HOST"
export ELASTIC_PORT="9243"
export ELASTIC_USER_NAME="elastic"
export ELASTIC_PASSWORD="$AZURE_ELASTIC_PASSWORD"
export DEFAULT_DATA_PARTITION_ID_TENANT1="opendes"
export DEFAULT_DATA_PARTITION_ID_TENANT2="common"
export ENVIRONMENT="CLOUD"
export ENTITLEMENTS_DOMAIN="contoso.com"
export LEGAL_TAG="opendes-public-usa-dataset-7643990"
export OTHER_RELEVANT_DATA_COUNTRIES="US"
```
__Azure Service Deployment__
1. Deploy the service using the Maven Plugin _(azure_deploy)_
```bash
cd $AZURE_BUILD_SUBDIR
mvn azure-webapp:deploy \
-DAZURE_TENANT_ID=$AZURE_TENANT_ID \
-Dazure.appservice.subscription=$AZURE_SUBSCRIPTION_ID \
-DAZURE_CLIENT_ID=$AZURE_CLIENT_ID \
-DAZURE_CLIENT_SECRET=$AZURE_CLIENT_SECRET \
-Dazure.appservice.resourcegroup=$AZURE_RESOURCE_GROUP \
-Dazure.appservice.plan=$AZURE_APPSERVICE_PLAN \
-Dazure.appservice.appname=$AZURE_APPSERVICE_NAME
```
2. Configure the Web App to start the SpringBoot Application _(azure_config)_
```bash
az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID
# Set the JAR FILE as required
TARGET=$(find ./target/ -name '*-spring-boot.jar')
JAR_FILE=${TARGET##*/}
JAVA_COMMAND="java -jar /home/site/wwwroot/${JAR_FILE}"
JSON_TEMPLATE='{"appCommandLine":"%s"}'
JSON_FILE="config.json"
echo $(printf "$JSON_TEMPLATE" "$JAVA_COMMAND") > $JSON_FILE
az webapp config set --resource-group $AZURE_RESOURCE_GROUP --name $AZURE_APPSERVICE_NAME --generic-configurations @$JSON_FILE
```
3. Execute the Integration Tests against the Service Deployment _(azure_test)_
```bash
mvn clean test -f $AZURE_TEST_SUBDIR/pom.xml
```
## License
......@@ -165,7 +263,7 @@ Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
......@@ -173,4 +271,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
\ No newline at end of file
limitations under the License.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment