Commit ade3d815 authored by harshit aggarwal's avatar harshit aggarwal
Browse files

Merge branch 'master' into azure/EventGridChanges

parents b9ea31a9 740c7d67
Pipeline #19992 failed with stages
in 2 minutes and 32 seconds
......@@ -8,6 +8,13 @@
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.releases.url>
</properties>
</profile>
......@@ -17,12 +24,20 @@
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
......
......@@ -6,7 +6,7 @@ from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
storage_account_name = os.environ.get('AZURE_STORAGE_ACCOUNT_NAME')
storage_account_key = os.environ.get('AZURE_STORAGE_ACCOUNT_KEY')
storage_account_url = "https://{}.blob.core.windows.net".format(storage_account_name)
container_name = os.environ.get('AZURE_STORAGE_CONTAINER')
container_name = os.environ.get('AZURE_MAPPINGS_STORAGE_CONTAINER')
blob_service_client = BlobServiceClient(account_url = storage_account_url,credential = storage_account_key)
local_folder = os.path.join(os.path.dirname(__file__), "../../mappings/opendes")
......
.bootstrap_variables:
variables:
AZURE_STORAGE_CONTAINER: osdu-wks-mappings
AZURE_MAPPINGS_STORAGE_CONTAINER: osdu-wks-mappings
AZURE_COSMOS_KEY: $AZURE_COSMOS_PRIMARY_KEY
AZURE_COSMOS_URL: $AZURE_COSMOS_CONNECTION
AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_BASE}data
......
parameters:
serviceName: 'wks'
testCoreMavenPomFile: ''
skipDeploy: false
skipTest: 'false'
environments: []
stages:
- ${{ each environment in parameters.environments }}:
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
testCoreMavenPomFile: ${{parameters.testCoreMavenPomFile}}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTest }}
providers:
- name: Azure
environments:
- ${{ environment }}
- stage: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
jobs:
- job: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
displayName: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
variables:
- group: 'Azure Target Env Secrets - ${{ environment }}'
- group: 'Azure Target Env - ${{ environment }}'
- group: 'Azure Service Release - ${{ parameters.serviceName }}'
- name: AZURE_DEPLOYMENTS_SUBDIR
value: deployments/scripts/azure
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploy Mappings'
inputs:
targetType: 'inline'
script: |
pip install -r $(AZURE_DEPLOYMENTS_SUBDIR)/requirements.txt
export STORAGE_CONTAINER=$(MAPPINGS_STORAGE_CONTAINER)
export AZURE_STORAGE_ACCOUNT_NAME=$(AZURE_STORAGE_ACCOUNT_NAME)
export AZURE_STORAGE_ACCOUNT_KEY=$(AZURE_STORAGE_ACCOUNT_KEY)
python $(AZURE_DEPLOYMENTS_SUBDIR)/deploy_mappings.py
\ No newline at end of file
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines an common set of supportive deploy stages that any external app service pipeline can call into.
#####################
parameters:
providers: []
serviceName: ''
hldRegPath: "providers/azure/hld-registry"
testCoreMavenPomFile: ''
testCoreMavenOptions: ''
integrationTestMavenGoal: 'package'
testCoreMavenGoal: 'install'
skipDeploy: false
skipTest: 'false'
dockerFile: ''
checkoutRepo: self
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Deploy_${{ provider.name }}_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
- group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
- name: artifactName
value: 'drop'
jobs:
- deployment: Deploy
pool:
name: $(AGENT_POOL)
demands:
- maven
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- download: none
- task: DownloadBuildArtifacts@0
displayName: 'Pull Build Artifacts'
inputs:
artifactName: '${{ variables.artifactName }}'
downloadPath: '$(System.DefaultWorkingDirectory)'
- template: deployment-steps.yml
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ provider.name }}
testCoreMavenPomFile: ${{ parameters.testCoreMavenPomFile }}
testCoreMavenGoal: ${{ parameters.testCoreMavenGoal }}
integrationTestMavenGoal: ${{ parameters.integrationTestMavenGoal }}
testCoreMavenOptions: ${{ parameters.testCoreMavenOptions }}
artifactName: ${{ variables.artifactName }}
elasticEndpoint: $(ELASTIC_ENDPOINT)
environment: ${{ environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
dockerFile: ${{ parameters.dockerFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTest }}
runPythonTest: ${{ parameters.runPythonTest }}
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
checkoutRepo: ${{ parameters.checkoutRepo }}
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines a common set of deployment steps that the deploy stage can call into.
#####################
parameters:
artifactName: 'drop'
serviceName: ''
providerName: ''
jarFilePattern: '*-SNAPSHOT*.jar'
integrationTestMavenGoal: 'package'
mavenPublishJUnitResults: false
testCoreMavenPomFile: ''
testCoreMavenGoal: 'install'
testCoreMavenOptions: ''
testArtifactLocation: 'drop/deploy'
elasticEndpoint: ''
skipTest: ''
dockerFile: ''
testPythonFilePath: ''
testPythonFile: ''
runPythonTest: ''
checkoutRepo: self
steps:
- template: /devops/tasks/detect-jar.yml@TemplateRepo
parameters:
stepName: DetectJarScript
providerName: ${{ parameters.providerName }}
pomFilePath: $(MAVEN_DEPLOY_POM_FILE_PATH)
jarFilePattern: ${{ parameters.jarFilePattern }}
- template: /devops/tasks/acr-publish.yml@TemplateRepo
parameters:
environment: ${{ parameters.environment }}
jarFilePath: $(DetectJarScript.JAR_FILE_PATH)
providerName: ${{ parameters.providerName }}
imageRepoName: ${{ parameters.serviceName }}
artifactName: ${{ parameters.artifactName }}
dockerFile: ${{ parameters.dockerFile }}
- template: /devops/tasks/aks-deployment-steps.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ parameters.providerName }}
environment: ${{ parameters.environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
checkoutRepo: ${{ parameters.checkoutRepo }}
- task: ExtractFiles@1
displayName: 'Extract ${{parameters.serviceName}} integration test suite '
inputs:
archiveFilePatterns: '$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/${{parameters.serviceName}}-integration-tests.zip'
destinationFolder: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}'
- template: /devops/tasks/elastic-setup.yml@TemplateRepo
parameters:
stepName: ExtractElasticHostAndPort
providerName: ${{ parameters.providerName }}
elasticEndpoint: ${{ parameters.elasticEndpoint }}
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploy Mappings'
inputs:
targetType: 'inline'
script: |
pip install -r $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/requirements.txt
export AZURE_MAPPINGS_STORAGE_CONTAINER=$(AZURE_MAPPINGS_STORAGE_CONTAINER)
export AZURE_STORAGE_ACCOUNT_NAME=$(STORAGE_ACCOUNT)
export AZURE_STORAGE_ACCOUNT_KEY=$(STORAGE_ACCOUNT_KEY)
export AZURE_COSMOS_KEY=$(AZURE_COSMOS_KEY)
export AZURE_COSMOS_URL=$(AZURE_COSMOS_URL)
python $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/deploy_mappings.py
python $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/deploy_mapping_info_records.py
- task: Maven@3
displayName: 'Maven build, test, and install ${{parameters.providerName}}-test-core'
condition: ne('${{ parameters.testCoreMavenPomFile }}', '')
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}/${{ parameters.testCoreMavenPomFile }}'
goals: ${{ parameters.testCoreMavenGoal }}
options: ${{ parameters.testCoreMavenOptions }}
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/flux-service-wait.yml@TemplateRepo
parameters:
serviceName: ${{parameters.serviceName}}
environment: ${{ parameters.environment }}
imageRepoName: '${{ parameters.serviceName }}'
- task: Maven@3
displayName: 'Maven run integration test'
condition: and(succeeded(), eq('${{ parameters.skipTest }}', 'false'))
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/$(MAVEN_INTEGRATION_TEST_POM_FILE_PATH)'
goals: ${{ parameters.integrationTestMavenGoal }}
options: $(MAVEN_INTEGRATION_TEST_OPTIONS)
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/python-integration-test.yml@TemplateRepo
parameters:
stepName: PythonRunIntegrationTests
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
runPythonTest: ${{ parameters.runPythonTest }}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/wks-gcp
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "wks"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
- name: 'MAVEN_CACHE_FOLDER'
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
value: 'false'
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P wks-core --settings .mvn/community-maven.settings.xml'
mavenOptions: '-P wks-azure --settings .mvn/community-maven.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/wks-azure/maven/settings.xml
provider/wks-azure/pom.xml
provider/wks-azure/target/*-spring-boot.jar
.mvn/community-maven.settings.xml
deployments/scripts/azure/*.py
deployments/mapping_info_records/opendes/*.json
deployments/mappings/opendes/*.json
deployments/scripts/azure/*.txt
copyFileContentsToFlatten: ''
mavenSettingsFile: '.mvn/community-maven.settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: deploy-stage.yml
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/wks-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven.settings.xml -DskipTests -DskipITs'
integrationTestMavenGoal: 'verify'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['dev']
......@@ -70,6 +70,7 @@ In order to run the service locally, you will need to have the following environ
| `OS_TARGET_SCHEMA_KIND` | ex `slb:oga:wellbore:1.0.0` | Schema kind for WKS records, takes a default value as (slb:wks:wellbore:1.3.1)| no | - |
| `OS_TARGET_SCHEMA_KIND_TENANT` | ex `opendes:oga:wellbore:1.0.0` | Schema kind for WKS records with tenant name, takes a default value as (opendes:wks:wellbore:1.3.1)| no | - |
| `DATA_PARTITION_ID` | ex `opendes` | data partition id| no | - |
| `TENANT_NAME` | ex `opendes` | tenant name | no | - |
### Build and run the application
......@@ -97,6 +98,21 @@ Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](htt
* `mvn clean install` and `mvn verfify` to be executed inside testing/wks-test-core directory (Make sure environment variable are set before executing these commands)
* Intellij can be used to run the tests using [Cucumber Plugin](https://plugins.jetbrains.com/plugin/7212-cucumber-for-java) for IntelliJ [ Use the EnvFile plugin to use environment variables directly from .env or .json/.yaml file]
### Bootstrapping
* There are two python scripts for bootstrapping mapping files and mapping info records. Mapping files will be uploaded
to Azure Blob Storage and mapping info records to Azure Cosmos Db.
* Commands to run the scripts can referenced from script section in [bootstrap.yml](../../devops/azure/bootstrap.yml)
* Environment variables required
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `AZURE_STORAGE_ACCOUNT_NAME` | `osdumvpdp1devqs29data` | storage account name | no | output of infrastructure deployment |
| `AZURE_STORAGE_ACCOUNT_KEY` | `********` | storage account key | yes | output of infrastructure deployment |
| `AZURE_MAPPINGS_STORAGE_CONTAINER` | `osdu-wks-mappings` | storage container where mappings are stored | no | output of infrastructure deployment |
| `AZURE_COSMOS_KEY` | `********` | cosmos account key | yes | output of infrastructure deployment |
| `AZURE_COSMOS_URL` | `********` | cosmos account endpoint | yes | output of infrastructure deployment |
## License
Copyright © Microsoft Corporation
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>azure-auth</id>
<configuration>
<tenant>${AZURE_DEPLOY_TENANT}</tenant>
<client>${AZURE_DEPLOY_CLIENT_ID}</client>
<key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
......@@ -39,7 +39,7 @@ public class JwtTokenGenerator implements UserCredential {
}
catch (UnsupportedEncodingException e) {
LOGGER.error(e.getMessage());
LOGGER.error(e.getMessage(), e);
throw new ApplicationException(e.getMessage(), e.getCause());
}
......
......@@ -64,12 +64,12 @@ public class ProcessWKSTransform {
} catch (BadRequestException e) {
LOGGER.error("Bad Request Reason: {}, pubsub message id: {}", e.getErrorMsg(),
message.getMessageId());
message.getMessageId(), e);
} catch (ApplicationException e) {
LOGGER.error("Application Error Reason: {}, pubsub message id: {}", e.getErrorMsg(),
message.getMessageId());
message.getMessageId(), e);
} catch (NullPointerException e) {
LOGGER.error("Invalid format for message with id: {}", message.getMessageId());
LOGGER.error("Invalid format for message with id: {}", message.getMessageId(), e);
}
catch (Exception e) {
LOGGER.error("Exception encountered processing the message with id: {}", message.getMessageId(), e);
......
......@@ -41,8 +41,14 @@ public class SubscriptionManagerImpl implements SubscriptionManager {
ExecutorService executorService = Executors
.newFixedThreadPool(Integer.parseUnsignedInt(azureBootstrapConfig.getNThreads()));
for (String partition : tenantList) {
SubscriptionClient subscriptionClient = this.subscriptionClientFactory.getSubscriptionClient(partition);
registerMessageHandler(subscriptionClient, executorService);
try {
SubscriptionClient subscriptionClient = this.subscriptionClientFactory.getSubscriptionClient(partition);
registerMessageHandler(subscriptionClient, executorService);
}
catch (Exception e) {
LOGGER.error("Error while creating or registering subscription client", e);
}
}
}
......@@ -55,7 +61,7 @@ public class SubscriptionManagerImpl implements SubscriptionManager {
executorService);
} catch (InterruptedException | ServiceBusException e) {
LOGGER.error("Error registering message handler {}", e.getMessage());
LOGGER.error("Error registering message handler {}", e.getMessage(), e);
}
}
......
......@@ -81,7 +81,7 @@ public class MappingStoreImpl implements MappingStore {
mappingsList.add(mapper.readValue(content, MappingsModel.class));
} catch (Exception e) {
LOGGER.error("Error while processing mappings from blob store {}", e.getMessage());
LOGGER.error("Error while processing mappings from blob store {}", e.getMessage(), e);
}
}
return mappingsList;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment