Commit 740c7d67 authored by Hema Vishnu Pola [Microsoft]'s avatar Hema Vishnu Pola [Microsoft]
Browse files

Merge branch 'azure/AddingDevPipeline' into 'master'

Adding dev pipeline

See merge request !28
parents 838a95b7 44769da3
Pipeline #19888 passed with stages
in 28 minutes
......@@ -8,6 +8,13 @@
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.releases.url>
</properties>
</profile>
......@@ -17,12 +24,20 @@
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
......
......@@ -6,7 +6,7 @@ from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
storage_account_name = os.environ.get('AZURE_STORAGE_ACCOUNT_NAME')
storage_account_key = os.environ.get('AZURE_STORAGE_ACCOUNT_KEY')
storage_account_url = "https://{}.blob.core.windows.net".format(storage_account_name)
container_name = os.environ.get('AZURE_STORAGE_CONTAINER')
container_name = os.environ.get('AZURE_MAPPINGS_STORAGE_CONTAINER')
blob_service_client = BlobServiceClient(account_url = storage_account_url,credential = storage_account_key)
local_folder = os.path.join(os.path.dirname(__file__), "../../mappings/opendes")
......
.bootstrap_variables:
variables:
AZURE_STORAGE_CONTAINER: osdu-wks-mappings
AZURE_MAPPINGS_STORAGE_CONTAINER: osdu-wks-mappings
AZURE_COSMOS_KEY: $AZURE_COSMOS_PRIMARY_KEY
AZURE_COSMOS_URL: $AZURE_COSMOS_CONNECTION
AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_BASE}data
......
parameters:
serviceName: 'wks'
testCoreMavenPomFile: ''
skipDeploy: false
skipTest: 'false'
environments: []
stages:
- ${{ each environment in parameters.environments }}:
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
testCoreMavenPomFile: ${{parameters.testCoreMavenPomFile}}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTest }}
providers:
- name: Azure
environments:
- ${{ environment }}
- stage: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
jobs:
- job: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
displayName: 'Deploy_Azure_WKS_Mappings_${{ environment }}'
variables:
- group: 'Azure Target Env Secrets - ${{ environment }}'
- group: 'Azure Target Env - ${{ environment }}'
- group: 'Azure Service Release - ${{ parameters.serviceName }}'
- name: AZURE_DEPLOYMENTS_SUBDIR
value: deployments/scripts/azure
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploy Mappings'
inputs:
targetType: 'inline'
script: |
pip install -r $(AZURE_DEPLOYMENTS_SUBDIR)/requirements.txt
export STORAGE_CONTAINER=$(MAPPINGS_STORAGE_CONTAINER)
export AZURE_STORAGE_ACCOUNT_NAME=$(AZURE_STORAGE_ACCOUNT_NAME)
export AZURE_STORAGE_ACCOUNT_KEY=$(AZURE_STORAGE_ACCOUNT_KEY)
python $(AZURE_DEPLOYMENTS_SUBDIR)/deploy_mappings.py
\ No newline at end of file
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines an common set of supportive deploy stages that any external app service pipeline can call into.
#####################
parameters:
providers: []
serviceName: ''
hldRegPath: "providers/azure/hld-registry"
testCoreMavenPomFile: ''
testCoreMavenOptions: ''
integrationTestMavenGoal: 'package'
testCoreMavenGoal: 'install'
skipDeploy: false
skipTest: 'false'
dockerFile: ''
checkoutRepo: self
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Deploy_${{ provider.name }}_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
- group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
- name: artifactName
value: 'drop'
jobs:
- deployment: Deploy
pool:
name: $(AGENT_POOL)
demands:
- maven
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- download: none
- task: DownloadBuildArtifacts@0
displayName: 'Pull Build Artifacts'
inputs:
artifactName: '${{ variables.artifactName }}'
downloadPath: '$(System.DefaultWorkingDirectory)'
- template: deployment-steps.yml
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ provider.name }}
testCoreMavenPomFile: ${{ parameters.testCoreMavenPomFile }}
testCoreMavenGoal: ${{ parameters.testCoreMavenGoal }}
integrationTestMavenGoal: ${{ parameters.integrationTestMavenGoal }}
testCoreMavenOptions: ${{ parameters.testCoreMavenOptions }}
artifactName: ${{ variables.artifactName }}
elasticEndpoint: $(ELASTIC_ENDPOINT)
environment: ${{ environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
dockerFile: ${{ parameters.dockerFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTest }}
runPythonTest: ${{ parameters.runPythonTest }}
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
checkoutRepo: ${{ parameters.checkoutRepo }}
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines a common set of deployment steps that the deploy stage can call into.
#####################
parameters:
artifactName: 'drop'
serviceName: ''
providerName: ''
jarFilePattern: '*-SNAPSHOT*.jar'
integrationTestMavenGoal: 'package'
mavenPublishJUnitResults: false
testCoreMavenPomFile: ''
testCoreMavenGoal: 'install'
testCoreMavenOptions: ''
testArtifactLocation: 'drop/deploy'
elasticEndpoint: ''
skipTest: ''
dockerFile: ''
testPythonFilePath: ''
testPythonFile: ''
runPythonTest: ''
checkoutRepo: self
steps:
- template: /devops/tasks/detect-jar.yml@TemplateRepo
parameters:
stepName: DetectJarScript
providerName: ${{ parameters.providerName }}
pomFilePath: $(MAVEN_DEPLOY_POM_FILE_PATH)
jarFilePattern: ${{ parameters.jarFilePattern }}
- template: /devops/tasks/acr-publish.yml@TemplateRepo
parameters:
environment: ${{ parameters.environment }}
jarFilePath: $(DetectJarScript.JAR_FILE_PATH)
providerName: ${{ parameters.providerName }}
imageRepoName: ${{ parameters.serviceName }}
artifactName: ${{ parameters.artifactName }}
dockerFile: ${{ parameters.dockerFile }}
- template: /devops/tasks/aks-deployment-steps.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ parameters.providerName }}
environment: ${{ parameters.environment }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
hldRegPath: ${{ parameters.hldRegPath }}
skipDeploy: ${{ parameters.skipDeploy }}
checkoutRepo: ${{ parameters.checkoutRepo }}
- task: ExtractFiles@1
displayName: 'Extract ${{parameters.serviceName}} integration test suite '
inputs:
archiveFilePatterns: '$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/${{parameters.serviceName}}-integration-tests.zip'
destinationFolder: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}'
- template: /devops/tasks/elastic-setup.yml@TemplateRepo
parameters:
stepName: ExtractElasticHostAndPort
providerName: ${{ parameters.providerName }}
elasticEndpoint: ${{ parameters.elasticEndpoint }}
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploy Mappings'
inputs:
targetType: 'inline'
script: |
pip install -r $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/requirements.txt
export AZURE_MAPPINGS_STORAGE_CONTAINER=$(AZURE_MAPPINGS_STORAGE_CONTAINER)
export AZURE_STORAGE_ACCOUNT_NAME=$(STORAGE_ACCOUNT)
export AZURE_STORAGE_ACCOUNT_KEY=$(STORAGE_ACCOUNT_KEY)
export AZURE_COSMOS_KEY=$(AZURE_COSMOS_KEY)
export AZURE_COSMOS_URL=$(AZURE_COSMOS_URL)
python $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/deploy_mappings.py
python $(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure/deploy_mapping_info_records.py
- task: Maven@3
displayName: 'Maven build, test, and install ${{parameters.providerName}}-test-core'
condition: ne('${{ parameters.testCoreMavenPomFile }}', '')
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}/${{ parameters.testCoreMavenPomFile }}'
goals: ${{ parameters.testCoreMavenGoal }}
options: ${{ parameters.testCoreMavenOptions }}
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/flux-service-wait.yml@TemplateRepo
parameters:
serviceName: ${{parameters.serviceName}}
environment: ${{ parameters.environment }}
imageRepoName: '${{ parameters.serviceName }}'
- task: Maven@3
displayName: 'Maven run integration test'
condition: and(succeeded(), eq('${{ parameters.skipTest }}', 'false'))
inputs:
mavenPomFile: '$(System.DefaultWorkingDirectory)/$(MAVEN_INTEGRATION_TEST_POM_FILE_PATH)'
goals: ${{ parameters.integrationTestMavenGoal }}
options: $(MAVEN_INTEGRATION_TEST_OPTIONS)
publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
- template: /devops/tasks/python-integration-test.yml@TemplateRepo
parameters:
stepName: PythonRunIntegrationTests
testPythonFilePath: ${{ parameters.testPythonFilePath }}
testPythonFile: ${{ parameters.testPythonFile }}
runPythonTest: ${{ parameters.runPythonTest }}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/wks-gcp
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "wks"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
- name: 'MAVEN_CACHE_FOLDER'
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
value: 'false'
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P wks-core --settings .mvn/community-maven.settings.xml'
mavenOptions: '-P wks-azure --settings .mvn/community-maven.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/wks-azure/maven/settings.xml
provider/wks-azure/pom.xml
provider/wks-azure/target/*-spring-boot.jar
.mvn/community-maven.settings.xml
deployments/scripts/azure/*.py
deployments/mapping_info_records/opendes/*.json
deployments/mappings/opendes/*.json
deployments/scripts/azure/*.txt
copyFileContentsToFlatten: ''
mavenSettingsFile: '.mvn/community-maven.settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: deploy-stage.yml
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/wks-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven.settings.xml -DskipTests -DskipITs'
integrationTestMavenGoal: 'verify'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['dev']
......@@ -70,6 +70,7 @@ In order to run the service locally, you will need to have the following environ
| `OS_TARGET_SCHEMA_KIND` | ex `slb:oga:wellbore:1.0.0` | Schema kind for WKS records, takes a default value as (slb:wks:wellbore:1.3.1)| no | - |
| `OS_TARGET_SCHEMA_KIND_TENANT` | ex `opendes:oga:wellbore:1.0.0` | Schema kind for WKS records with tenant name, takes a default value as (opendes:wks:wellbore:1.3.1)| no | - |
| `DATA_PARTITION_ID` | ex `opendes` | data partition id| no | - |
| `TENANT_NAME` | ex `opendes` | tenant name | no | - |
### Build and run the application
......@@ -97,6 +98,21 @@ Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](htt
* `mvn clean install` and `mvn verfify` to be executed inside testing/wks-test-core directory (Make sure environment variable are set before executing these commands)
* Intellij can be used to run the tests using [Cucumber Plugin](https://plugins.jetbrains.com/plugin/7212-cucumber-for-java) for IntelliJ [ Use the EnvFile plugin to use environment variables directly from .env or .json/.yaml file]
### Bootstrapping
* There are two python scripts for bootstrapping mapping files and mapping info records. Mapping files will be uploaded
to Azure Blob Storage and mapping info records to Azure Cosmos Db.
* Commands to run the scripts can referenced from script section in [bootstrap.yml](../../devops/azure/bootstrap.yml)
* Environment variables required
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `AZURE_STORAGE_ACCOUNT_NAME` | `osdumvpdp1devqs29data` | storage account name | no | output of infrastructure deployment |
| `AZURE_STORAGE_ACCOUNT_KEY` | `********` | storage account key | yes | output of infrastructure deployment |
| `AZURE_MAPPINGS_STORAGE_CONTAINER` | `osdu-wks-mappings` | storage container where mappings are stored | no | output of infrastructure deployment |
| `AZURE_COSMOS_KEY` | `********` | cosmos account key | yes | output of infrastructure deployment |
| `AZURE_COSMOS_URL` | `********` | cosmos account endpoint | yes | output of infrastructure deployment |
## License
Copyright © Microsoft Corporation
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>azure-auth</id>
<configuration>
<tenant>${AZURE_DEPLOY_TENANT}</tenant>
<client>${AZURE_DEPLOY_CLIENT_ID}</client>
<key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment