Commit c38ab13a authored by Daniel Scholl's avatar Daniel Scholl
Browse files

Added in Azure Devops and Pipeline automation

parent 2ffbc135
Pipeline #10906 failed with stage
in 3 minutes and 50 seconds
......@@ -38,6 +38,7 @@ load-tests/*.pyc
# Environment configuration
*.env
.sts4*
.envrc
# Intellij module setting file
*.iml
......@@ -47,4 +48,4 @@ load-tests/*.pyc
.DS_STORE
dist/
\ No newline at end of file
dist/
......@@ -4,6 +4,11 @@ variables:
AWS_SERVICE: partition
AWS_ENVIRONMENT: dev
AZURE_SERVICE: partition
AZURE_BUILD_SUBDIR: provider/partition-azure
AZURE_TEST_SUBDIR: testing/partition-test-azure
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
......@@ -18,4 +23,7 @@ include:
file: "scanners/fossa.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws.yml'
\ No newline at end of file
file: 'cloud-providers/aws.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure.yml"
# Pipeline Support Commands
```bash
AZURE_SERVICE="partition"
REPO_BRANCH="master"
TAG="latest"
PARTIAL=${REPO_BRANCH/\//-}
BRANCH=${PARTIAL/./-}
echo "--set image.branch=$BRANCH --set image.tag=$TAG"
# Install the Service
helm upgrade -i osdu-gitlab-$AZURE_SERVICE chart --set image.branch=$BRANCH --set image.tag=$TAG
pod=$(kubectl get pod |grep $AZURE_SERVICE | tail -1 | awk '{print $1}')
status=$(kubectl wait --for=condition=Ready pod/$pod --timeout=60s)
if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
```
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v2
name: partition
appVersion: "latest"
description: Helm Chart for installing storage service.
version: 0.1.0
type: application
# This file contains the essential configs for the osdu on azure helm chart
global:
# Service(s) Replica Count
replicaCount: 2
################################################################################
# Specify the Gitlab branch being used for image creation
# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
#
image:
repository: ${CONTAINER_REGISTRY_NAME}.azurecr.io
branch: ${ENVIRONMENT_NAME}
tag: ${BUILD_BUILDID}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}
namespace: osdu
spec:
replicas: {{ .Values.global.replicaCount }}
selector:
matchLabels:
app: {{ .Chart.Name }}
template:
metadata:
labels:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/partition/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: partition
- name: server.servlet.contextPath
value: /api/partition/v1/
- name: server_port
value: "80"
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: azure_activedirectory_session_stateless
value: "true"
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: {{ .Chart.Name }}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
global:
replicaCount: 1
image:
repository: community.opengroup.org:5555/osdu/platform/system/partition
branch: master
tag: latest
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/partition-aws
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "partition"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P partition-core'
mavenOptions: '-P partition-azure'
copyFileContents: |
pom.xml
provider/partition-azure/maven/settings.xml
provider/partition-azure/pom.xml
provider/partition-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/partition-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['dev']
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/partition-aws
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "partition"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P partition-core'
mavenOptions: '-P partition-azure'
copyFileContents: |
pom.xml
provider/partition-azure/maven/settings.xml
provider/partition-azure/pom.xml
provider/partition-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/partition-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['demo']
---
# Source: partition/templates/service.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: partition
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: partition
---
# Source: partition/templates/deployment.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: osdu-gitlab-partition
namespace: osdu
spec:
replicas: 1
selector:
matchLabels:
app: partition
template:
metadata:
labels:
app: partition
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: partition
image: community.opengroup.org:5555/osdu/platform/system/partition/partition-trusted-azure-pipeline:latest
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/partition/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: partition
- name: server.servlet.contextPath
value: /api/partition/v1/
- name: server_port
value: "80"
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: azure_activedirectory_session_stateless
value: "true"
......@@ -30,12 +30,35 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `KEYVAULT_URI` | (non-secret) | KeyVault URI | no | variable `AZURE_KEYVAULT_URI` from GitLab variable group `Azure Target Env - {{env}}`
| `appinsights_key` | `********` | Application Insights Instrumentation Key, required to hook AppInsights with Partition application | yes | keyvault secret: `$KEYVAULT_URI/secrets/appinsights-key` |
| `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
| `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
| `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
| `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
| `KEYVAULT_URI` | (non-secret) | KeyVault URI | no | variable `AZURE_KEYVAULT_URI` from GitLab variable group `Azure Target Env - {{env}}` |
| `aad_client_id` | `********` | AAD client application ID | yes | keyvault secret: `$KEYVAULT_URI/secrets/aad-client-id` |
| `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
| `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
| `appinsights_key` | `********` | Application Insights Instrumentation Key, required to hook AppInsights with Partition application | yes | keyvault secret: `$KEYVAULT_URI/secrets/appinsights-key` |
**Required to run integration tests**
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `PARTITION_BASE_URL` | ex `http://localhost:8080/` | The host where the service is running. NO CONTEXT! | no | -- |
| `ENVIRONMENT` | ex `LOCAL` | The environment name | no | LOCAL/HOSTED |
| `MY_TENANT` | ex `opendes` | OSDU tenant used for testing | no | -- |
| `CLIENT_TENANT` | ex `common` | Client tenant used for testing | no | -- |
| `DEFAULT_PARTITION` | ex `opendes` | Default Tenant Name used bypasses Data Preperation and Teardown of tests | no | -- |
| `AZURE_AD_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | -- |
| `INTEGRATION_TESTER` | `********` | System identity to assume for API calls. Note: this user must have entitlements configured already | no | -- |
| `AZURE_TESTER_SERVICEPRINCIPAL_SECRET` | `********` | Secret for `$INTEGRATION_TESTER` | yes | -- |
| `AZURE_AD_APP_RESOURCE_ID` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `AZURE_AD_OTHER_APP_RESOURCE_ID` | `********` | AAD client application ID for another application | yes | -- |
| `NO_DATA_ACCESS_TESTER` | `********` | Service principal ID of a service principal without entitlements | yes | `aad-no-data-access-tester-client-id` secret from keyvault |
| `NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET` | `********` | Secret for `$NO_DATA_ACCESS_TESTER` | yes | `aad-no-data-access-tester-secret` secret from keyvault |
### Configure Maven
......@@ -48,21 +71,6 @@ Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b0
...
```
You will need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`:
```bash
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>os-core</id>
<username>os-core</username>
<password>${VSTS_FEED_TOKEN}</password>
</server>
</servers>
</settings>
```
### Build and run the application
......@@ -73,30 +81,30 @@ After configuring your environment as specified above, you can follow these step
$ mvn clean install
# build + test + package azure service code
$ mvn clean package -P partition-aks
$ (cd provider/partition-azure/ && mvn clean package)
# run service
#
# Note: this assumes that the environment variables for running the service as outlined
# above are already exported in your environment.
$ cd provider/partition-azure && mvn spring-boot:run -f pom.xml
$ java -jar $(find provider/partition-azure/target/ -name '*-spring-boot.jar')
```
### Test the application
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/api/partition/v1/swagger-ui.html](http://localhost:8080/api/partition/v1/swagger-ui.html). If the request does not fail, you can then run the integration tests.
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests.
see [instructions](../../testing/README.md) on how to run integration tests locally. In addition to common testing environment variables, the `partition-test-azure` module also needs additional environment variables, which are described below:
```bash
# build + install integration test core
$ (cd testing/partition-test-core/ && mvn clean install)