diff --git a/devops/custom_pipelines/charts/osdu-common.yml b/devops/custom_pipelines/charts/osdu-common.yml new file mode 100644 index 0000000000000000000000000000000000000000..3187386f84588ecbf3e196d56c607db12502f4c9 --- /dev/null +++ b/devops/custom_pipelines/charts/osdu-common.yml @@ -0,0 +1,69 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /pipelines/charts/osdu-common.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: HelmRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /charts/osdu-common/* + exclude: + - /**/*.md + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU secrets' + + - name: serviceName + value: "osdu-common" + - name: chartPath + value: "charts/osdu-common" + - name: valuesFile + value: "charts/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['HelmRepo'].name ] + +stages: + - template: /devops/chart-stages.yml@HelmRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + skipDeploy: ${{ variables.SKIP_DEPLOY }} + checkoutRepo: HelmRepo + providers: + - name: Azure + environments: ["dev"] diff --git a/devops/custom_pipelines/charts/osdu-istio-auth.yml b/devops/custom_pipelines/charts/osdu-istio-auth.yml new file mode 100644 index 0000000000000000000000000000000000000000..50fd26e7c132906e55d18872c2638cf5847840eb --- /dev/null +++ b/devops/custom_pipelines/charts/osdu-istio-auth.yml @@ -0,0 +1,70 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /infra-chart-pipelines/chart-osdu-istio-auth.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: HelmRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /charts/osdu-istio-auth/* + exclude: + - /**/*.md + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU secrets' + + - name: serviceName + value: "osdu-istio-auth" + - name: chartPath + value: "charts/osdu-istio-auth" + - name: valuesFile + value: "charts/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['HelmRepo'].name ] + +stages: + - template: /devops/chart-stages.yml@HelmRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipCheck: true + checkoutRepo: HelmRepo + providers: + - name: Azure + environments: ["dev"] diff --git a/devops/custom_pipelines/charts/osdu-istio.yml b/devops/custom_pipelines/charts/osdu-istio.yml new file mode 100644 index 0000000000000000000000000000000000000000..ad3a1c96bdbb4c062c90bd72460279c97cb590df --- /dev/null +++ b/devops/custom_pipelines/charts/osdu-istio.yml @@ -0,0 +1,70 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /infra-chart-pipelines/chart-osdu-istio.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: HelmRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /charts/osdu-istio/* + exclude: + - /**/*.md + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU secrets' + + - name: serviceName + value: "osdu-istio" + - name: chartPath + value: "charts/osdu-istio" + - name: valuesFile + value: "charts/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['HelmRepo'].name ] + +stages: + - template: /devops/chart-stages.yml@HelmRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipCheck: true + checkoutRepo: HelmRepo + providers: + - name: Azure + environments: ["dev"] diff --git a/devops/custom_pipelines/infrastructure/common-resource.yml b/devops/custom_pipelines/infrastructure/common-resource.yml new file mode 100644 index 0000000000000000000000000000000000000000..5c92a3f0b0ba1c8447580545ce614a2474f37275 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/common-resource.yml @@ -0,0 +1,73 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /pipelines/infrastructure/common-resource.yml + +resources: + repositories: + - repository: InfraRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /devops/* + - /infra/modules/providers/* + - /infra/templates/osdu-r3-mvp/central_resources/* + - /test-harness/terratest-extensions/* + exclude: + - /**/*.md + +variables: +- name: 'TF_VERSION' + value: '0.12.29' +- name: 'GO_VERSION' + value: '1.12.14' +- name: 'TF_ROOT_DIR' + value: 'infra' +- name: 'PIPELINE_ROOT_DIR' + value: 'devops/infrastructure' +- name: 'TEST_HARNESS_DIR' + value: 'test-harness' +- group: 'Infrastructure Pipeline Variables' +- name: 'INFRA_REPO' + value: $[ resources.repositories['InfraRepo'].name ] + +stages: +- template: devops/infra-prepare-stage.yml@InfraRepo + parameters: + checkoutRepo: InfraRepo + environments: + - name: 'dev' + enablePrIsolation: false + resourceNameIsolationLevel: 8 + + configurations: + - jobName: central_resources + terraformWorkspacePrefix: cr + terraformTemplatePath: infra/templates/osdu-r3-mvp/central_resources + terraformVariableFile: pipelines/infrastructure/common-resources.tfvars + deploymentTimeoutInMinutes: 120 diff --git a/devops/custom_pipelines/infrastructure/common-resources.tfvars b/devops/custom_pipelines/infrastructure/common-resources.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..7a0e39fa89ada839970368ff119a760802a98ac1 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/common-resources.tfvars @@ -0,0 +1,24 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Configuration +.DESCRIPTION + This file holds Override Variable Configuration +*/ + +resource_tags = { + contact = "OSDU Development" +} diff --git a/devops/custom_pipelines/infrastructure/data-partition.tfvars b/devops/custom_pipelines/infrastructure/data-partition.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..4a3ae414f7d1559473ffc9f977d211aafb25ef19 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/data-partition.tfvars @@ -0,0 +1,32 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Configuration +.DESCRIPTION + This file holds Override Variable Configuration +*/ + +resource_tags = { + contact = "OSDU Development" +} + +# Database Settings +cosmos_databases = [ + { + name = "osdu-db" + throughput = 8000 + } +] diff --git a/devops/custom_pipelines/infrastructure/data-partition.yml b/devops/custom_pipelines/infrastructure/data-partition.yml new file mode 100644 index 0000000000000000000000000000000000000000..b28f634100a8228afad7f51024661e7b51368a21 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/data-partition.yml @@ -0,0 +1,73 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /pipelines/infrastructure/data-partition.yml + +resources: + repositories: + - repository: InfraRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /infra/modules/providers/* + - /infra/templates/osdu-r3-mvp/data_partition/* + - /test-harness/terratest-extensions/* + exclude: + - /**/*.md + +variables: +- name: 'TF_VERSION' + value: '0.12.29' +- name: 'GO_VERSION' + value: '1.12.14' +- name: 'TF_ROOT_DIR' + value: 'infra' +- name: 'PIPELINE_ROOT_DIR' + value: 'devops' +- name: 'TEST_HARNESS_DIR' + value: 'test-harness' +- group: 'Infrastructure Pipeline Variables' +- name: 'INFRA_REPO' + value: $[ resources.repositories['InfraRepo'].name ] + +stages: +- template: /devops/infra-prepare-stage.yml@InfraRepo + parameters: + checkoutRepo: InfraRepo + environments: + - name: 'dev' + enablePrIsolation: false + resourceNameIsolationLevel: 8 + + configurations: + - jobName: data_partition_1 + terraformWorkspacePrefix: dp1 + terraformTemplatePath: /infra/templates/osdu-r3-mvp/data_partition + terraformVariableFile: pipelines/infrastructure/data-partition.tfvars + deploymentTimeoutInMinutes: 120 + dataPartitionName: opendes diff --git a/devops/custom_pipelines/infrastructure/service-resource.tfvars b/devops/custom_pipelines/infrastructure/service-resource.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..812c2f151cbd36aa6c03509aac18b935256912c7 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/service-resource.tfvars @@ -0,0 +1,29 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Configuration +.DESCRIPTION + This file holds Override Variable Configuration +*/ + +resource_tags = { + contact = "OSDU Development" +} + +# Kubernetes Settings +kubernetes_version = "1.18.8" +aks_agent_vm_size = "Standard_E4s_v3" +aks_agent_vm_count = "5" +subnet_aks_prefix = "10.10.2.0/23" diff --git a/devops/custom_pipelines/infrastructure/service-resource.yml b/devops/custom_pipelines/infrastructure/service-resource.yml new file mode 100644 index 0000000000000000000000000000000000000000..c61bc11443a37e2fdd70ea533f74b2d2cea836c3 --- /dev/null +++ b/devops/custom_pipelines/infrastructure/service-resource.yml @@ -0,0 +1,73 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /pipelines/infrastructure/service-resource.yml + +resources: + repositories: + - repository: InfraRepo + type: git + name: infra-azure-provisioning + trigger: + branches: + include: + - master + paths: + include: + - /devops/infrastructure/* + - /infra/modules/providers/* + - /infra/templates/osdu-r3-mvp/service_resources/* + - /test-harness/terratest-extensions/* + exclude: + - /**/*.md + +variables: +- name: 'TF_VERSION' + value: '0.12.29' +- name: 'GO_VERSION' + value: '1.12.14' +- name: 'TF_ROOT_DIR' + value: 'infra' +- name: 'PIPELINE_ROOT_DIR' + value: 'devops/infrastructure' +- name: 'TEST_HARNESS_DIR' + value: 'test-harness' +- group: 'Infrastructure Pipeline Variables' +- name: 'INFRA_REPO' + value: $[ resources.repositories['InfraRepo'].name ] + +stages: +- template: devops/infra-prepare-stage.yml@InfraRepo + parameters: + checkoutRepo: InfraRepo + environments: + - name: 'dev' + enablePrIsolation: false + resourceNameIsolationLevel: 8 + + configurations: + - jobName: service_resources + terraformWorkspacePrefix: sr + terraformTemplatePath: infra/templates/osdu-r3-mvp/service_resources + terraformVariableFile: pipelines/infrastructure/service-resource.tfvars + deploymentTimeoutInMinutes: 120 diff --git a/devops/custom_pipelines/services/entitlements-azure.yml b/devops/custom_pipelines/services/entitlements-azure.yml new file mode 100644 index 0000000000000000000000000000000000000000..0d7aea6ade04b473bcc4e83f89e30fde6775b9e4 --- /dev/null +++ b/devops/custom_pipelines/services/entitlements-azure.yml @@ -0,0 +1,89 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /platform/system/entitlements.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + - repository: MavenRepo + type: git + name: entitlements-azure + ref: master + trigger: + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "entitlements-azure" + - name: branchName + value: master + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['MavenRepo'].name ] + - name: SKIP_TESTS + value: 'false' + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + copyFileContents: | + pom.xml + maven/settings.xml + target/*.jar + copyFileContentsToFlatten: '' + mavenOptions: '' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'integration-tests' + chartPath: ${{ variables.chartPath }} + checkoutRepo: MavenRepo + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + checkoutRepo: MavenRepo + providers: + - name: Azure + environments: ['dev'] diff --git a/devops/custom_pipelines/services/indexer-service.yml b/devops/custom_pipelines/services/indexer-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..e10558f63478f413f83078e3b612cfd35f325137 --- /dev/null +++ b/devops/custom_pipelines/services/indexer-service.yml @@ -0,0 +1,98 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /platform/system/indexer-service.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + - repository: MavenRepo + type: git + name: indexer-service + trigger: + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + - /docs + - /indexer/storage-aws + - /indexer/storage-byoc + - /indexer/storage-gcp + - /indexer/storage-ibm + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "indexer-service" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: SKIP_TESTS + value: 'false' + - name: 'HELM_SOURCE' + value: $[ resources.repositories['MavenRepo'].name ] + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P indexer-core' + mavenOptions: '-P indexer-azure' + copyFileContents: | + pom.xml + provider/indexer-azure/maven/settings.xml + provider/indexer-azure/pom.xml + provider/indexer-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + checkoutRepo: MavenRepo + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + checkoutRepo: MavenRepo + providers: + - name: Azure + environments: ['dev'] diff --git a/devops/custom_pipelines/services/legal.yml b/devops/custom_pipelines/services/legal.yml new file mode 100644 index 0000000000000000000000000000000000000000..1937966d65661b1fcb876cb336d47838494b7514 --- /dev/null +++ b/devops/custom_pipelines/services/legal.yml @@ -0,0 +1,91 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /platform/system/legal.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + - repository: MavenRepo + type: git + name: legal + trigger: + branches: + include: + - master + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "legal" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: SKIP_TESTS + value: 'true' + - name: MAVEN_CACHE_FOLDER + value: "$(HOME)/.m2/repository" + - name: 'HELM_SOURCE' + value: $[ resources.repositories['MavenRepo'].name ] + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P legal-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + mavenOptions: '-P legal-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + copyFileContents: | + pom.xml + provider/legal-azure/maven/settings.xml + provider/legal-azure/pom.xml + provider/legal-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + checkoutRepo: MavenRepo + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/legal-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + checkoutRepo: MavenRepo + providers: + - name: Azure + environments: ['dev'] diff --git a/devops/custom_pipelines/services/partition.yml b/devops/custom_pipelines/services/partition.yml new file mode 100644 index 0000000000000000000000000000000000000000..c59609e0b2a3f1c83008bc777329967fb97543a6 --- /dev/null +++ b/devops/custom_pipelines/services/partition.yml @@ -0,0 +1,98 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /platform/system/partition.yml + + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + - repository: MavenRepo + type: git + name: partition + trigger: + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + - /docs + - /provider/partition-aws + + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "partition" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['MavenRepo'].name ] + + +stages: + + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P partition-core' + mavenOptions: '-P partition-azure' + copyFileContents: | + pom.xml + provider/partition-azure/maven/settings.xml + provider/partition-azure/pom.xml + provider/partition-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + checkoutRepo: MavenRepo + + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/partition-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + checkoutRepo: MavenRepo + providers: + - name: Azure + environments: ['perf'] diff --git a/devops/custom_pipelines/services/storage.yml b/devops/custom_pipelines/services/storage.yml new file mode 100644 index 0000000000000000000000000000000000000000..55c7f39635bb578cbf2cb2076c06c4b98d9e209b --- /dev/null +++ b/devops/custom_pipelines/services/storage.yml @@ -0,0 +1,98 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample custom Pipeline. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /platform/system/storage.yml + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + - repository: MavenRepo + type: git + name: storage + trigger: + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + - /docs + - /provider/storage-aws + - /provider/storage-byoc + - /provider/storage-gcp + - /provider/storage-ibm + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "storage" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: 'HELM_SOURCE' + value: $[ resources.repositories['MavenRepo'].name ] + - name: SKIP_TESTS + value: 'false' + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P storage-core' + mavenOptions: '-P storage-azure' + copyFileContents: | + pom.xml + provider/storage-azure/maven/settings.xml + provider/storage-azure/pom.xml + provider/storage-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + checkoutRepo: MavenRepo + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/storage-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + checkoutRepo: MavenRepo + providers: + - name: Azure + environments: ['dev'] diff --git a/devops/infra-build-stage.yml b/devops/infra-build-stage.yml index 4c496410fa4c40359f1a3c7056e0803ef85a6f6a..77bb8bc2dad76ba406ba415d6385a928e97445b4 100644 --- a/devops/infra-build-stage.yml +++ b/devops/infra-build-stage.yml @@ -61,6 +61,9 @@ jobs: - name: TF_VAR_DATA_PARTITION_NAME value: ${{ parameters.configuration.dataPartitionName }} + - name: TF_VARFILE + value: '$(System.DefaultWorkingDirectory)/${{ parameters.configuration.terraformVariableFile }}' + workspace: clean: all diff --git a/devops/infra-deploy-stage.yml b/devops/infra-deploy-stage.yml index f2f240cdb42ee0554c7c55bbfb512c00dc4b223e..f1e37eb095c8901134cc59764d56e3b4449b6a18 100644 --- a/devops/infra-deploy-stage.yml +++ b/devops/infra-deploy-stage.yml @@ -19,6 +19,7 @@ parameters: skipTests: false aksAgentSshPublicKeyFilename: 'azure-aks-node-ssh-key.pub' aksGitOpsSshPrivateKeyFilename: 'azure-aks-gitops-ssh-key' + terraformVariablePath: '' jobs: @@ -65,6 +66,9 @@ jobs: - name: TF_VAR_DATA_PARTITION_NAME value: ${{ parameters.configuration.dataPartitionName }} + - name: TF_VARFILE + value: '$(System.DefaultWorkingDirectory)/${{ parameters.configuration.terraformVariableFile }}' + environment: ${{ parameters.environment }} strategy: runOnce: @@ -90,6 +94,7 @@ jobs: aksAgentSshPublicKeyFilename: ${{ parameters.aksAgentSshPublicKeyFilename }} aksGitOpsSshPrivateKeyFilename: ${{ parameters.aksGitOpsSshPrivateKeyFilename }} terraformWorkspacePrefix: ${{ parameters.configuration.terraformWorkspacePrefix }} + terraformVariablePath: ${{ parameters.configuration.terraformVariablePath }} - template: tasks/tests-int.yml parameters: diff --git a/devops/tasks/tf-apply.yml b/devops/tasks/tf-apply.yml index 2967fb2244cb3fabfe0a7db116deb234a011f4fb..a1a8765d2c0b826ca775081e2f67f2e5f681cf6b 100644 --- a/devops/tasks/tf-apply.yml +++ b/devops/tasks/tf-apply.yml @@ -32,6 +32,7 @@ steps: displayName: 'TF Apply' env: TERRAFORM_WORKSPACE_PREFIX: ${{ parameters.terraformWorkspacePrefix }} + TF_VAR_override_file: $(System.DefaultWorkingDirectory)/${{ parameters.terraformVariableFile }} inputs: azureSubscription: '$(SERVICE_CONNECTION_NAME)' addSpnToEnvironment: true @@ -73,11 +74,12 @@ steps: TF_PLAN_FILE="${TF_WORKSPACE_NAME}_plan.out" TF_CLI_ARGS=${TF_CLI_ARGS:-} + [ ! -f $TF_VARFILE ] && TF_VARFILE=${TF_TEMPLATE_WORKING_DIR}/override.tfvars echo "TF_WORKSPACE_NAME: ${TF_WORKSPACE_NAME}" echo "TF_VAR_DATA_PARTITION_NAME: ${TF_VAR_DATA_PARTITION_NAME}" - terraform apply $TF_CLI_ARGS -input=false -auto-approve $TF_PLAN_FILE + terraform apply -var-file $TF_VAFILE $TF_CLI_ARGS -input=false -auto-approve $TF_PLAN_FILE # Remove the flux folder auto-generated from bedrock rm -rf --interactive=never ${TERRAFORM_WORKSPACE_PREFIX,,}* diff --git a/devops/tasks/tf-plan.yml b/devops/tasks/tf-plan.yml index 637ff570a08bdd4a69e6c13b0380767dcffbaec0..ea02e39f2fdb1ed218978dde530d4a3f6dbfa8a5 100644 --- a/devops/tasks/tf-plan.yml +++ b/devops/tasks/tf-plan.yml @@ -56,6 +56,7 @@ steps: TF_VAR_principal_password: $(TF_VAR_principal_password) TF_VAR_principal_objectId: $(TF_VAR_principal_objectId) TF_VAR_gitops_path: $(TF_VAR_gitops_path) + TF_VAR_override_file: $(System.DefaultWorkingDirectory)/${{ parameters.terraformVariableFile }} inputs: azureSubscription: '$(SERVICE_CONNECTION_NAME)' @@ -100,10 +101,12 @@ steps: TF_PLAN_FILE="${TF_WORKSPACE_NAME}_plan.out" TF_CLI_ARGS=${TF_CLI_ARGS:-} + [ ! -f $TF_VARFILE ] && TF_VARFILE=${TF_TEMPLATE_WORKING_DIR}/override.tfvars echo "TF_WORKSPACE_NAME: ${TF_WORKSPACE_NAME}" echo "TF_VAR_DATA_PARTITION_NAME: ${TF_VAR_DATA_PARTITION_NAME}" + echo "TF_VARFILE: ${TF_VARFILE}" echo "File 1: $(aksPublicAgentKeySecureDownload.secureFilePath)" echo "File 2: $(aksPrivateGitopsKeySecureDownload.secureFilePath)" - terraform plan $TF_CLI_ARGS -input=false -out $TF_PLAN_FILE + terraform plan -var-file $TF_VAFILE $TF_CLI_ARGS -input=false -out $TF_PLAN_FILE diff --git a/infra/templates/osdu-r3-mvp/central_resources/override.tfvars b/infra/templates/osdu-r3-mvp/central_resources/override.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..eeb5e301fa0226dab05bc35fedd115096f25edd7 --- /dev/null +++ b/infra/templates/osdu-r3-mvp/central_resources/override.tfvars @@ -0,0 +1,30 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Override Template +.DESCRIPTION + This file holds a sample of variable overrides to be used by pipelines +*/ + +/* + +prefix = "osdu-mvp" + +resource_tags = { + environment = "OSDU Demo" +} + +*/ diff --git a/infra/templates/osdu-r3-mvp/data_partition/override.tfvars b/infra/templates/osdu-r3-mvp/data_partition/override.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..4593b4f442f9925c0d87a559fabcd5c78bc322a0 --- /dev/null +++ b/infra/templates/osdu-r3-mvp/data_partition/override.tfvars @@ -0,0 +1,37 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Override Template +.DESCRIPTION + This file holds a sample of variable overrides to be used by pipelines +*/ + +/* + +prefix = "osdu-mvp" + +resource_tags = { + environment = "OSDU Demo" +} + +cosmos_databases = [ + { + name = "osdu-db" + throughput = 12000 + } +] + +*/ diff --git a/infra/templates/osdu-r3-mvp/service_resources/override.tfvars b/infra/templates/osdu-r3-mvp/service_resources/override.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..3bcca0c380992c69acbef82c9a29ccccd1f37af1 --- /dev/null +++ b/infra/templates/osdu-r3-mvp/service_resources/override.tfvars @@ -0,0 +1,36 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +.Synopsis + Terraform Variable Override Template +.DESCRIPTION + This file holds a variable override sample to be used by pipelines +*/ + +/* + +prefix = "osdu-mvp" + +resource_tags = { + environment = "OSDU Demo" +} + +# Kubernetes Settings +kubernetes_version = "1.18.8" +aks_agent_vm_size = "Standard_E4s_v3" +aks_agent_vm_count = "5" +subnet_aks_prefix = "10.10.2.0/23" + +*/