Commit be10ac3b authored by David Diederich's avatar David Diederich
Browse files

Merge remote-tracking branch 'origin/master' into release/0.4

parents 631b977e ca324b8f
Pipeline #20734 failed with stages
in 119 minutes and 40 seconds
......@@ -37,3 +37,5 @@ target/
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
*local.properties
dist/
\ No newline at end of file
......@@ -16,9 +16,11 @@ variables:
AZURE_BUILD_SUBDIR: provider/schema-azure
AZURE_TEST_SUBDIR: testing/schema-test-core
AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR: deployments/scripts
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
DATA_PARTITION: opendes
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=${AZURE_BASE}data;AccountKey=${AZURE_STORAGE_KEY};EndpointSuffix=core.windows.net
......@@ -31,7 +33,7 @@ variables:
OSDU_GCP_SERVICE: schema
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-schema
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,ACCOUNT_ID_COMMON_PROJECT=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
......@@ -69,4 +71,4 @@ include:
ref: "v0.4.3"
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- local: "/devops/azure/bootstrap.yaml"
- local: "/devops/azure/gitlab-bootstrap.yml"
......@@ -524,7 +524,7 @@ The following software have components provided under the terms of this license:
- jakarta.xml.bind-api (from )
========================================================================
CC-BY-3.0
CC-BY-2.5
========================================================================
The following software have components provided under the terms of this license:
......@@ -552,7 +552,6 @@ CDDL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- JavaMail API (from )
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
......@@ -562,14 +561,21 @@ CDDL-1.1
The following software have components provided under the terms of this license:
- JavaBeans Activation Framework (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- tomcat-embed-core (from http://tomcat.apache.org/)
========================================================================
EPL-1.0
CPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
========================================================================
EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- Logback Classic Module (from )
- Logback Core Module (from )
......@@ -666,7 +672,6 @@ LGPL-2.1-or-later
========================================================================
The following software have components provided under the terms of this license:
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -773,7 +778,6 @@ Public-Domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
......@@ -797,6 +801,7 @@ public-domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Asynchronous Http Client (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
......@@ -811,6 +816,7 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- StAX API (from http://stax.codehaus.org/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
......@@ -821,6 +827,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- JUnit (from http://junit.org)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
......
......@@ -5,9 +5,9 @@ class AzureToken(object):
def get_azure_id_token(self):
tenant_id = os.getenv('AZURE_TENANT_ID')
resource_id = os.getenv('AZURE_APP_ID')
client_id = os.getenv('INTEGRATION_TESTER')
client_secret = os.getenv('AZURE_TESTER_SERVICEPRINCIPAL_SECRET')
resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
client_id = os.getenv('AZURE_CLIENT_ID')
client_secret = os.getenv('AZURE_CLIENT_SECRET')
if tenant_id is None:
print('Please pass tenant Id to generate token')
......
export AZURE_SCHEMA_URL=https://$AZURE_DNS_NAME/api/schema-service/v1/schema
BEARER_TOKEN=`python $AZURE_DEPLOYMENTS_SUBDIR/Token.py`
export BEARER_TOKEN=$BEARER_TOKEN
python deployments/scripts/DeploySharedSchemas.py -u $AZURE_SCHEMA_URL
python $AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $AZURE_SCHEMA_URL
\ No newline at end of file
parameters:
artifactName: 'drop'
serviceName: ''
providers: []
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Bootstrap_${{ provider.name }}_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
- group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
- name: artifactName
value: 'drop'
jobs:
- deployment: Bootstrap
pool:
name: $(AGENT_POOL)
demands:
- maven
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- download: none
- task: DownloadBuildArtifacts@0
displayName: 'Pull Build Artifacts'
inputs:
artifactName: '${{ variables.artifactName }}'
downloadPath: '$(System.DefaultWorkingDirectory)'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Bootstrap Schemas'
inputs:
targetType: 'inline'
script: |
export AZURE_DEPLOYMENTS_SUBDIR=$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure
# These variables are required for generating the token
export AZURE_TENANT_ID=$(AZURE_TENANT_ID)
export AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)
export AZURE_CLIENT_ID=$(INTEGRATION_TESTER)
export AZURE_CLIENT_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)
export DATA_PARTITION=$(DATA_PARTITION_ID)
pip install -r $AZURE_DEPLOYMENTS_SUBDIR/requirements.txt
chmod +x $AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh
$AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file contains the essential configs for the osdu on azure helm chart
global:
# Service(s) Replica Count
replicaCount: 1
################################################################################
# Specify the Gitlab branch being used for image creation
# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
#
image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
......@@ -29,70 +29,73 @@ spec:
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/schema-service/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: schema
- name: LOG_PREFIX
value: "schema"
- name: server.servlet.contextPath
value: /api/schema-service/v1/
- name: server.port
value: "80"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: cosmosdb_database
value: osdu-db
- name: storage_account
value: osdumvpdp1glabky7vdata
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
- name: entitlements_service_api_key
value: "OBSOLETE"
\ No newline at end of file
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/schema-service/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: schema
- name: LOG_PREFIX
value: "schema"
- name: server.servlet.contextPath
value: /api/schema-service/v1/
- name: server.port
value: "80"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: storage_account
valueFrom:
secretKeyRef:
name: storage
key: storage-account
- name: cosmosdb_database
value: osdu-db
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
- name: entitlements_service_api_key
value: "OBSOLETE"
\ No newline at end of file
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines an common set of supportive deploy stages that any external app service pipeline can call into.
#####################
parameters:
providers: []
serviceName: ''
hldRegPath: "providers/azure/hld-registry"
testCoreMavenPomFile: ''
testCoreMavenOptions: ''
integrationTestMavenGoal: 'package'
testCoreMavenGoal: 'install'
skipDeploy: false
skipTest: 'false'
dockerFile: ''
checkoutRepo: self
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Deploy_${{ provider.name }}_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
- group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
- name: artifactName
value: 'drop'
jobs:
- deployment: Deploy
pool:
name: $(AGENT_POOL)
demands:
- maven
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- download: none
- task: DownloadBuildArtifacts@0
displayName: 'Pull Build Artifacts'
inputs:
artifactName: '${{ variables.artifactName }}'
downloadPath: '$(System.DefaultWorkingDirectory)'
- template: deployment-steps.yml
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ provider.name }}
testCoreMavenPomFile: ${{ parameters.testCoreMavenPomFile }}
testCoreMavenGoal: ${{ parameters.testCoreMavenGoal }}
integrationTestMavenGoal: ${{ parameters.integrationTestMavenGoal }}
testCoreMavenOptions: ${{ parameters.testCoreMavenOptions }}
artifactName: ${{ variables.artifactName }}
elasticEndpoint: $(ELASTIC_ENDPOINT)
environment: ${{ environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
dockerFile: ${{ parameters.dockerFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTest }}
runPythonTest: ${{ parameters.runPythonTest }}
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
checkoutRepo: ${{ parameters.checkoutRepo }}
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines a common set of deployment steps that the deploy stage can call into.
#####################
parameters:
artifactName: 'drop'
serviceName: ''
providerName: ''
jarFilePattern: '*-SNAPSHOT*.jar'
integrationTestMavenGoal: 'package'
mavenPublishJUnitResults: false
testCoreMavenPomFile: ''
testCoreMavenGoal: 'install'
testCoreMavenOptions: ''
testArtifactLocation: 'drop/deploy'
elasticEndpoint: ''
skipTest: ''
dockerFile: ''
testPythonFilePath: ''
testPythonFile: ''
runPythonTest: ''
checkoutRepo: self
steps:
- template: /devops/tasks/detect-jar.yml@TemplateRepo
parameters:
stepName: DetectJarScript
providerName: ${{ parameters.providerName }}
pomFilePath: $(MAVEN_DEPLOY_POM_FILE_PATH)
jarFilePattern: ${{ parameters.jarFilePattern }}
- template: /devops/tasks/acr-publish.yml@TemplateRepo
parameters:
environment: ${{ parameters.environment }}
jarFilePath: $(DetectJarScript.JAR_FILE_PATH)
providerName: ${{ parameters.providerName }}
imageRepoName: ${{ parameters.serviceName }}
artifactName: ${{ parameters.artifactName }}
dockerFile: ${{ parameters.dockerFile }}
- template: /devops/tasks/aks-deployment-steps.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ parameters.providerName }}
environment: ${{ parameters.environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
checkoutRepo: ${{ parameters.checkoutRepo }}
- task: ExtractFiles@1
displayName: 'Extract ${{parameters.serviceName}} integration test suite '
inputs:
archiveFilePatterns: '$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/${{parameters.serviceName}}-integration-tests.zip'
destinationFolder: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}'
- template: /devops/tasks/elastic-setup.yml@TemplateRepo
parameters:
stepName: ExtractElasticHostAndPort
providerName: ${{ parameters.providerName }}
elasticEndpoint: ${{ parameters.elasticEndpoint }}
- task: Maven@3
displayName: 'Maven build, test, and install ${{parameters.providerName}}-test-core'
condition: ne('${{ parameters.testCoreMavenPomFile }}', '')
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}/${{ parameters.testCoreMavenPomFile }}'
goals: ${{ parameters.testCoreMavenGoal }}
options: ${{ parameters.testCoreMavenOptions }}
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/flux-service-wait.yml@TemplateRepo
parameters:
serviceName: ${{parameters.serviceName}}
environment: ${{ parameters.environment }}
imageRepoName: '${{ parameters.serviceName }}'
- task: Maven@3
displayName: 'Maven run integration test'
condition: and(succeeded(), eq('${{ parameters.skipTest }}', 'false'))
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/$(MAVEN_INTEGRATION_TEST_POM_FILE_PATH)'
goals: ${{ parameters.integrationTestMavenGoal }}
options: $(MAVEN_INTEGRATION_TEST_OPTIONS)
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/python-integration-test.yml@TemplateRepo
parameters:
stepName: PythonRunIntegrationTests
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
runPythonTest: ${{ parameters.runPythonTest }}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/schema-aws
- /provider/schema-gcp
- /provider/schema-ibm
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "schema-service"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]