Merge branch 'aks' into 'master'

port aks and data pipelines to OSDU

See merge request !6

data/devops/azure/pipeline.yml

+trigger:
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - /data
+    exclude:
+      - README.md
+      - .gitignore
+      - LICENSE
+      - NOTICE
+      - pom.xml
+      - skaffold.yaml
+      - /crs-catalog-core
+      - /devops
+      - /maven
+      - /provider
+      - /testing
+
+variables:
+  - group: 'Azure - OSDU'
+
+stages:
+  - template: stages.yml
+    parameters:
+      environments:
+        - name: 'dev'
+        # - ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+        #   - name: 'qa'
+        #   - name: 'prod'
+

data/devops/azure/stages.yml

+parameters:
+  environments: []
+
+stages:
+  - ${{ each environment in parameters.environments }}:
+    - stage: 'Build_${{ environment.name }}'
+
+      variables:
+      - group: 'Infrastructure Pipeline Variables - ${{environment.name}}'
+      - group: 'Azure Target Env Secrets - ${{environment.name}}'
+      - group: 'Azure Target Env - ${{environment.name}}' 
+
+      jobs:
+        - job: MavenPackageAndPublishArtifacts
+          displayName: Build Package and Publish Artifacts
+          
+          pool: $(AGENT_POOL)
+
+          steps:
+          - download: none
+
+          - checkout: self
+            path: s
+
+          - task: AzureCLI@1
+            displayName: 'Docker Build + ACR Push'
+            # condition: and(succeeded(), eq('${{ parameters.providerName }}', 'Azure'))
+            env:
+              IMAGE: $(CONTAINER_REGISTRY_NAME).azurecr.io/crs-catalog-data:v2
+            inputs:
+              azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+              addSpnToEnvironment: true
+              scriptLocation: inlineScript
+              inlineScript: |
+                #!/usr/bin/env bash
+                set -euo pipefail
+
+                curl -L https://aka.ms/acr/installaad/bash | /bin/bash
+                echo "Logging in to the ACR Registry"
+                echo "------------------------------------"
+                az acr login -n $(CONTAINER_REGISTRY_NAME)
+
+                pushd data
+                docker build -t $IMAGE .
+                docker push $IMAGE
+                popd

devops/azure/chart/helm-config.yaml

+# This file contains the essential configs for the osdu on azure helm chart
+global:
+
+  # Service(s) Replica Count
+  replicaCount: 2
+
+################################################################################
+# Specify the Gitlab branch being used for image creation
+# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
+#
+image:
+  repository: #{container-registry}#.azurecr.io
+  branch: #{ENVIRONMENT_NAME}#
+  tag: #{Build.SourceVersion}#
+
+catalog:
+  repository: #{container-registry}#.azurecr.io/crs-catalog-data
+  tag: v2
+  pullPolicy: Always  

devops/azure/chart/templates/deployment.yaml

@@ -23,8 +23,6 @@ spec:
             secretProviderClass: azure-keyvault
       - name: shared-data
         emptyDir: {}
-      imagePullSecrets:
-        - name: {{ .Values.image.imagePullSecrets }}         
       initContainers:
       - name: "init-crs-catalog"
         image: "{{ .Values.catalog.repository }}:{{ .Values.catalog.tag }}"
@@ -34,8 +32,8 @@ spec:
           mountPath: /mnt/crs_catalogs              
       containers:
       - name: {{ .Chart.Name }}
-        image: "{{ .Values.app.repository }}:{{ .Values.app.tag }}"
-        imagePullPolicy: {{ .Values.app.pullPolicy }}
+        image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+        imagePullPolicy: Always
         ports:
         - containerPort: 80
         readinessProbe:
@@ -63,30 +61,30 @@ spec:
               name: osdu-svc-properties
               key: ENV_KEYVAULT
         - name: AZURE_TENANT_ID
-          valueFrom:
-            configMapKeyRef:
-              name: osdu-svc-properties
-              key: ENV_TENANT_ID
-        - name: client-id
           valueFrom:
             secretKeyRef:
-              name: clientid
-              key: clientid
-        - name: AZURE_CLIENT_SECRET
+              name: active-directory
+              key: tenantid
+        - name: AZURE_CLIENT_ID
           valueFrom:
             secretKeyRef:
-              name: clientpassword
-              key: clientpassword
-        - name: aad_client_id
+              name: active-directory
+              key: principal-clientid
+        - name: AZURE_CLIENT_SECRET
           valueFrom:
             secretKeyRef:
-              name: appid
-              key: appid
+              name: active-directory
+              key: principal-clientpassword
         - name: appinsights_key
           valueFrom:
             secretKeyRef:
-              name: appinsights
+              name: central-logging
               key: appinsights
+        - name: aad_client_id
+          valueFrom:
+            secretKeyRef:
+              name: active-directory
+              key: application-appid
         - name: ENTITLEMENT_URL
           value: http://entitlements-azure/entitlements/v1
         - name: azure_istioauth_enabled

devops/azure/chart/values.yaml

@@ -2,16 +2,12 @@ global:
   replicaCount: 1
 
 image:
-  imagePullSecrets: acr
-
-app:
-  repository: community.opengroup.org:5555/osdu/platform/ref-and-helper/crs-catalog-app
-  branch: master
+  repository: ${CONTAINER_REGISTRY_NAME}.azurecr.io/crs-catalog-app
   tag: latest
   pullPolicy: Always
 
 catalog:
-  repository: community.opengroup.org:5555/osdu/platform/ref-and-helper/crs-catalogs
+  repository: ${CONTAINER_REGISTRY_NAME}.azurecr.io/crs-catalog-data
   tag: v2
   pullPolicy: Always
 

devops/azure/pipeline.yml

+
+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    exclude:
+      - /**/*.md
+      - .gitignore
+      - /docs
+
+
+resources:
+  repositories:
+  - repository: FluxRepo
+    type: git
+    name: gitops-manifests
+  - repository: TemplateRepo
+    type: git
+    name: infra-azure-provisioning
+
+variables:
+  - group: 'Azure - OSDU'
+  - group: 'Azure - OSDU Secrets'
+
+  - name: serviceName
+    value: "crs-catalog-service"
+  - name: chartPath
+    value: "devops/azure/chart"
+  - name: valuesFile
+    value: "devops/azure/chart/helm-config.yaml"
+  - name: 'MANIFEST_REPO'
+    value: $[ resources.repositories['FluxRepo'].name ]
+
+
+stages:
+
+  - template: /devops/build-stage.yml@TemplateRepo
+    parameters:
+      mavenGoal: 'package'
+      mavenPublishJUnitResults: true
+      serviceCoreMavenOptions: ''
+      mavenOptions: ''
+      copyFileContents: |
+        pom.xml
+        provider/crs-catalog-azure/crs-catalog-aks/pom.xml
+        provider/crs-catalog-azure/crs-catalog-aks/target/*.jar
+      copyFileContentsToFlatten: ''
+      mavenSettingsFile: './maven/settings.xml'
+      serviceBase: ${{ variables.serviceName }}
+      testingRootFolder: 'testing'
+      chartPath: ${{ variables.chartPath }}
+
+  - template: /devops/deploy-stages.yml@TemplateRepo
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      chartPath: ${{ variables.chartPath }}
+      valuesFile: ${{ variables.valuesFile }}
+      testCoreMavenPomFile: ''
+      testCoreMavenOptions: ''
+      skipDeploy: ${{ variables.SKIP_DEPLOY }}
+      skipTest: 'true'
+      providers:
+        -  name: Azure
+           environments: ['dev']

pom.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
@@ -28,12 +28,12 @@
     	</license>
     </licenses>
 
-    <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
-    <artifactId>crs-catalog-service</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-    <name>crs-catalog-service</name>
-    <description>crs-catalog service</description>
+	<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
+	<artifactId>crs-catalog-service</artifactId>
+	<version>1.0.0</version>
+	<packaging>pom</packaging>
+	<name>crs-catalog-service</name>
+	<description>crs-catalog service</description>
 
     <modules>
         <module>crs-catalog-core</module>
@@ -51,11 +51,11 @@
     <distributionManagement>
         <repository>
             <id>${gitlab-server}</id>
-            <url>https://community.opengroup.org/api/v4/projects/74/packages/maven</url>
+            <url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
         </repository>
         <snapshotRepository>
             <id>${gitlab-server}</id>
-            <url>https://community.opengroup.org/api/v4/projects/74/packages/maven</url>
+            <url>https://community.opengroup.org/api/v4/projects/21/packages/maven</url>
         </snapshotRepository>
     </distributionManagement>
 

provider/crs-catalog-azure/crs-catalog-aks/pom.xml

@@ -6,14 +6,14 @@
 
 
     <parent>
-        <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
-        <artifactId>crs-catalog-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
+		<groupId>org.opengroup.osdu.crs-catalog-service</groupId>
+		<artifactId>crs-catalog-service</artifactId>
+		<version>1.0.0</version>
+	</parent>
 
     <groupId>org.opengroup.osdu.crs-catalog-service</groupId>
     <artifactId>crs-catalog-aks</artifactId>
-    <version>1.0.0</version>
+    <version>1.0.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>crs-catalog-aks</name>
     <description>CRS Catalog service Google Kubernetes Engine deployment</description>
@@ -38,7 +38,7 @@
 
     <properties>
         <app.version>1</app.version>
-        <app.id>crs-catalog-aks</app.id>
+        <app.id>crs-catalog-aks</app.id>    
 
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
@@ -46,7 +46,7 @@
         <azure.version>2.1.7</azure.version>
         <slf4j.version>1.7.25</slf4j.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
+        <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>	    
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
     </properties>
@@ -65,15 +65,15 @@
 
         <!-- Compile/runtime dependencies -->
         <dependency>
-            <groupId>javax.jdo</groupId>
-            <artifactId>jdo-api</artifactId>
-            <version>3.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.8.5</version>
-        </dependency>
+        <groupId>javax.jdo</groupId>
+        <artifactId>jdo-api</artifactId>
+        <version>3.1</version>
+        </dependency>        
+		<dependency>
+	        <groupId>com.google.code.gson</groupId>
+	        <artifactId>gson</artifactId>
+	        <version>2.8.5</version>
+		</dependency>
 
         <dependency>
             <groupId>org.projectlombok</groupId>
@@ -125,10 +125,10 @@
         <!-- for hot reload of the web application-->
         <outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>
         <plugins>
-            <plugin>
-                <groupId>org.springframework.boot</groupId>
-                <artifactId>spring-boot-maven-plugin</artifactId>
-            </plugin>
+          <plugin>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-maven-plugin</artifactId>
+          </plugin>
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>versions-maven-plugin</artifactId>
@@ -144,7 +144,7 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
+               <groupId>org.apache.maven.plugins</groupId>
                 <version>3.7.0</version>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>