port aks and data pipelines to OSDU

6f98c9bd · Sherman Yang · eff7d3a9 · 6f98c9bd · 6f98c9bd · 6f98c9bd
Commit 6f98c9bd authored Oct 09, 2020 by Sherman Yang
--- a/data/devops/azure/pipeline.yml
+++ b/data/devops/azure/pipeline.yml
+trigger:
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - /data
+    exclude:
+      - README.md
+      - .gitignore
+      - LICENSE
+      - NOTICE
+      - pom.xml
+      - skaffold.yaml
+      - /crs-catalog-core
+      - /devops
+      - /maven
+      - /provider
+      - /testing
+
+variables:
+  - group: 'Azure - OSDU'
+
+stages:
+  - template: stages.yml
+    parameters:
+      environments:
+        - name: 'dev'
+        # - ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+        #   - name: 'qa'
+        #   - name: 'prod'
+
--- a/data/devops/azure/stages.yml
+++ b/data/devops/azure/stages.yml
+parameters:
+  environments: []
+
+stages:
+  - ${{ each environment in parameters.environments }}:
+    - stage: 'Build_${{ environment.name }}'
+
+      variables:
+      - group: 'Infrastructure Pipeline Variables - ${{environment.name}}'
+      - group: 'Azure Target Env Secrets - ${{environment.name}}'
+      - group: 'Azure Target Env - ${{environment.name}}' 
+
+      jobs:
+        - job: MavenPackageAndPublishArtifacts
+          displayName: Build Package and Publish Artifacts
+          
+          pool: $(AGENT_POOL)
+
+          steps:
+          - download: none
+
+          - checkout: self
+            path: s
+
+          - task: AzureCLI@1
+            displayName: 'Docker Build + ACR Push'
+            # condition: and(succeeded(), eq('${{ parameters.providerName }}', 'Azure'))
+            env:
+              IMAGE: $(CONTAINER_REGISTRY_NAME).azurecr.io/crs-catalog-data:v2
+            inputs:
+              azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+              addSpnToEnvironment: true
+              scriptLocation: inlineScript
+              inlineScript: |
+                #!/usr/bin/env bash
+                set -euo pipefail
+
+                curl -L https://aka.ms/acr/installaad/bash | /bin/bash
+                echo "Logging in to the ACR Registry"
+                echo "------------------------------------"
+                az acr login -n $(CONTAINER_REGISTRY_NAME)
+
+                pushd data
+                docker build -t $IMAGE .
+                docker push $IMAGE
+                popd
--- a/devops/azure/chart/helm-config.yaml
+++ b/devops/azure/chart/helm-config.yaml
+# This file contains the essential configs for the osdu on azure helm chart
+global:
+
+  # Service(s) Replica Count
+  replicaCount: 2
+
+################################################################################
+# Specify the Gitlab branch being used for image creation
+# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
+#
+image:
+  repository: #{container-registry}#.azurecr.io
+  branch: #{ENVIRONMENT_NAME}#
+  tag: #{Build.SourceVersion}#
+
+catalog:
+  repository: #{container-registry}#.azurecr.io/crs-catalog-data
+  tag: v2
+  pullPolicy: Always  
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -23,8 +23,6 @@ spec:
            secretProviderClass: azure-keyvault
      - name: shared-data
        emptyDir: {}
-      imagePullSecrets:
-        - name: {{ .Values.image.imagePullSecrets }}         
      initContainers:
      - name: "init-crs-catalog"
        image: "{{ .Values.catalog.repository }}:{{ .Values.catalog.tag }}"
@@ -34,8 +32,8 @@ spec:
          mountPath: /mnt/crs_catalogs              
      containers:
      - name: {{ .Chart.Name }}
-        image: "{{ .Values.app.repository }}:{{ .Values.app.tag }}"
-        imagePullPolicy: {{ .Values.app.pullPolicy }}
+        image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+        imagePullPolicy: Always
        ports:
        - containerPort: 80
        readinessProbe:
@@ -63,30 +61,30 @@ spec:
              name: osdu-svc-properties
              key: ENV_KEYVAULT
        - name: AZURE_TENANT_ID
-          valueFrom:
-            configMapKeyRef:
-              name: osdu-svc-properties
-              key: ENV_TENANT_ID
-        - name: client-id
          valueFrom:
            secretKeyRef:
-              name: clientid
-              key: clientid
-        - name: AZURE_CLIENT_SECRET
+              name: active-directory
+              key: tenantid
+        - name: AZURE_CLIENT_ID
          valueFrom:
            secretKeyRef:
-              name: clientpassword
-              key: clientpassword
-        - name: aad_client_id
+              name: active-directory
+              key: principal-clientid
+        - name: AZURE_CLIENT_SECRET
          valueFrom:
            secretKeyRef:
-              name: appid
-              key: appid
+              name: active-directory
+              key: principal-clientpassword
        - name: appinsights_key
          valueFrom:
            secretKeyRef:
-              name: appinsights
+              name: central-logging
              key: appinsights
+        - name: aad_client_id
+          valueFrom:
+            secretKeyRef:
+              name: active-directory
+              key: application-appid
        - name: ENTITLEMENT_URL
          value: http://entitlements-azure/entitlements/v1
        - name: azure_istioauth_enabled

--- a/devops/azure/chart/values.yaml
+++ b/devops/azure/chart/values.yaml
@@ -2,16 +2,12 @@ global:
  replicaCount: 1

 image:
-  imagePullSecrets: acr
-
-app:
-  repository: community.opengroup.org:5555/osdu/platform/ref-and-helper/crs-catalog-app
-  branch: master
+  repository: ${CONTAINER_REGISTRY_NAME}.azurecr.io/crs-catalog-app
  tag: latest
  pullPolicy: Always

 catalog:
-  repository: community.opengroup.org:5555/osdu/platform/ref-and-helper/crs-catalogs
+  repository: ${CONTAINER_REGISTRY_NAME}.azurecr.io/crs-catalog-data
  tag: v2
  pullPolicy: Always


--- a/devops/azure/pipeline.yml
+++ b/devops/azure/pipeline.yml
+
+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    exclude:
+      - /**/*.md
+      - .gitignore
+      - /docs
+
+
+resources:
+  repositories:
+  - repository: FluxRepo
+    type: git
+    name: gitops-manifests
+  - repository: TemplateRepo
+    type: git
+    name: infra-azure-provisioning
+
+variables:
+  - group: 'Azure - OSDU'
+  - group: 'Azure - OSDU Secrets'
+
+  - name: serviceName
+    value: "crs-catalog-service"
+  - name: chartPath
+    value: "devops/azure/chart"
+  - name: valuesFile
+    value: "devops/azure/chart/helm-config.yaml"
+  - name: 'MANIFEST_REPO'
+    value: $[ resources.repositories['FluxRepo'].name ]
+
+
+stages:
+
+  - template: /devops/build-stage.yml@TemplateRepo
+    parameters:
+      mavenGoal: 'package'
+      mavenPublishJUnitResults: true
+      serviceCoreMavenOptions: ''
+      mavenOptions: ''
+      copyFileContents: |
+        pom.xml
+        provider/crs-catalog-azure/crs-catalog-aks/pom.xml
+        provider/crs-catalog-azure/crs-catalog-aks/target/*.jar
+      copyFileContentsToFlatten: ''
+      mavenSettingsFile: './maven/settings.xml'
+      serviceBase: ${{ variables.serviceName }}
+      testingRootFolder: 'testing'
+      chartPath: ${{ variables.chartPath }}
+
+  - template: /devops/deploy-stages.yml@TemplateRepo
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      chartPath: ${{ variables.chartPath }}
+      valuesFile: ${{ variables.valuesFile }}
+      testCoreMavenPomFile: ''
+      testCoreMavenOptions: ''
+      skipDeploy: ${{ variables.SKIP_DEPLOY }}
+      skipTest: 'true'
+      providers:
+        -  name: Azure
+           environments: ['dev']
--- a/pom.xml
+++ b/pom.xml
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
@@ -28,12 +28,12 @@
    	</license>
    </licenses>

-    <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
-    <artifactId>crs-catalog-service</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-    <name>crs-catalog-service</name>
-    <description>crs-catalog service</description>
+	<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
+	<artifactId>crs-catalog-service</artifactId>
+	<version>1.0.0</version>
+	<packaging>pom</packaging>
+	<name>crs-catalog-service</name>
+	<description>crs-catalog service</description>

    <modules>
        <module>crs-catalog-core</module>
@@ -51,11 +51,11 @@
    <distributionManagement>
        <repository>
            <id>${gitlab-server}</id>
-            <url>https://community.opengroup.org/api/v4/projects/74/packages/maven</url>
+            <url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
        </repository>
        <snapshotRepository>
            <id>${gitlab-server}</id>
-            <url>https://community.opengroup.org/api/v4/projects/74/packages/maven</url>
+            <url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
        </snapshotRepository>
    </distributionManagement>


--- a/provider/crs-catalog-azure/crs-catalog-aks/pom.xml
+++ b/provider/crs-catalog-azure/crs-catalog-aks/pom.xml
@@ -6,14 +6,14 @@


    <parent>
-        <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
-        <artifactId>crs-catalog-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
+		<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
+		<artifactId>crs-catalog-service</artifactId>
+		<version>1.0.0</version>
+	</parent>

    <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
    <artifactId>crs-catalog-aks</artifactId>
-    <version>1.0.0</version>
+    <version>1.0.0-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>crs-catalog-aks</name>
    <description>CRS Catalog service Google Kubernetes Engine deployment</description>
@@ -38,7 +38,7 @@

    <properties>
        <app.version>1</app.version>
-        <app.id>crs-catalog-aks</app.id>
+        <app.id>crs-catalog-aks</app.id>    

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

@@ -46,7 +46,7 @@
        <azure.version>2.1.7</azure.version>
        <slf4j.version>1.7.25</slf4j.version>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
+        <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>	    
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
    </properties>
@@ -65,15 +65,15 @@

        <!-- Compile/runtime dependencies -->
        <dependency>
-            <groupId>javax.jdo</groupId>
-            <artifactId>jdo-api</artifactId>
-            <version>3.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.8.5</version>
-        </dependency>
+        <groupId>javax.jdo</groupId>
+        <artifactId>jdo-api</artifactId>
+        <version>3.1</version>
+        </dependency>        
+		<dependency>
+	        <groupId>com.google.code.gson</groupId>
+	        <artifactId>gson</artifactId>
+	        <version>2.8.5</version>
+		</dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
@@ -125,10 +125,10 @@
        <!-- for hot reload of the web application-->
        <outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>
        <plugins>
-            <plugin>
-                <groupId>org.springframework.boot</groupId>
-                <artifactId>spring-boot-maven-plugin</artifactId>
-            </plugin>
+          <plugin>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-maven-plugin</artifactId>
+          </plugin>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>versions-maven-plugin</artifactId>
@@ -144,7 +144,7 @@
                </executions>
            </plugin>
            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
+               <groupId>org.apache.maven.plugins</groupId>
                <version>3.7.0</version>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>