Merge branch 'master' of...

Merge branch 'master' of https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/os-indexer into kuber-migration

devops/service-pipeline.yml

+#####################
+# README: Defines a template to be used as a starting point for defining a service pipeline
+#####################
+
 trigger:
   batch: true
   branches:
     include:
-    - master
+      - master
   paths:
     exclude:
-    - /**/*.md
-    - .gitignore
-    - images/
+      - /**/*.md
+      - .gitignore
+      - images/
 
 pr:
+  autoCancel: true
   branches:
     include:
-    - '*'
+      - '*'
   paths:
     exclude:
-    - /**/*.md
-    - .gitignore
-    - images/
+      - /**/*.md
+      - .gitignore
+      - images/
+
+variables:
+  - group: 'Azure Common Secrets'
+  - group: 'Azure - Common'
+
+  - name: serviceName
+    value: 'indexer'
 
 resources:
   repositories:
@@ -25,9 +37,31 @@ resources:
       type: git
       name: open-data-ecosystem/infrastructure-templates
 
-variables:
-- group: 'Azure Common Secrets'
-- group: 'Azure - Common'
-
 stages:
-- template: devops/service-pipelines/service-pipeline-stages.yml@infrastructure-templates
+  - template: devops/service-pipelines/build-stage.yml@infrastructure-templates
+    parameters:
+      mavenGoal: 'package'
+      mavenPublishJUnitResults: true
+      serviceCoreMavenOptions: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U'
+      mavenOptions: '--settings ./provider/${{ variables.serviceName }}-azure/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U -P indexer-azure'
+      copyFileContents: |
+          pom.xml
+          provider/${{ variables.serviceName }}-azure/maven/settings.xml
+          provider/${{ variables.serviceName }}-azure/pom.xml
+          provider/${{ variables.serviceName }}-azure/target/*-spring-boot.jar
+      copyFileContentsToFlatten: ''
+      serviceBase: ${{ variables.serviceName }}
+      testingRootFolder: 'testing'
+  - template: devops/service-pipelines/deploy-stages.yml@infrastructure-templates
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml'
+      testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U'
+      providers:
+        -  name: Azure
+           # Merges into Master
+           ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+            environments: ['devint', 'qa', 'prod']
+           # PR updates / creations
+           ${{ if ne(variables['Build.SourceBranchName'], 'master') }}:
+            environments: ['devint']

indexer-core/pom.xml

@@ -18,7 +18,7 @@
 		<dependency>
 			<groupId>org.opengroup.osdu</groupId>
 			<artifactId>os-core-common</artifactId>
-			<version>0.0.10-SNAPSHOT</version>
+			<version>0.0.12</version>
 		</dependency>
 
 		<!-- spring boot dependencies -->

provider/indexer-aws/pom.xml

@@ -39,7 +39,7 @@
       <dependency>
           <groupId>org.opengroup.osdu</groupId>
           <artifactId>os-core-common</artifactId>
-          <version>0.0.10-SNAPSHOT</version>
+          <version>0.0.12</version>
       </dependency>
     <dependency>
         <groupId>org.opengroup.osdu.indexer</groupId>
@@ -49,7 +49,7 @@
     <dependency>
         <groupId>org.opengroup.osdu.core.aws</groupId>
         <artifactId>aws-osdu-util</artifactId>
-        <version>0.0.6-SNAPSHOT</version>
+        <version>0.0.8</version>
     </dependency>
 
     <!-- AWS managed packages -->

provider/indexer-azure/README.md

@@ -15,3 +15,8 @@ Due to the usage of [Managed Identity](https://docs.microsoft.com/en-us/azure/ac
 | `AZURE_CLIENT_ID`     | service principal's app id                          |
 | `AZURE_TENANT_ID`     | id of the principal's Azure Active Directory tenant |
 | `AZURE_CLIENT_SECRET` | one of the service principal's client secrets       |
+
+## Deploying service to Azure
+
+Service deployments into Azure are standardized to make the process the same for all services. The steps to deploy into
+Azure can be [found here](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/infrastructure-templates?path=%2Fdocs%2Fosdu%2FSERVICE_DEPLOYMENTS.md&_a=preview)

provider/indexer-azure/pom.xml

@@ -88,7 +88,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-common</artifactId>
-            <version>0.0.10-SNAPSHOT</version>
+            <version>0.0.12</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu.indexer</groupId>
@@ -99,7 +99,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-azure</artifactId>
-            <version>0.0.6-SNAPSHOT</version>
+            <version>0.0.8</version>
         </dependency>
 
         <dependency>

provider/indexer-gcp/pom.xml

@@ -38,13 +38,13 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-gcp</artifactId>
-            <version>0.1.14-SNAPSHOT</version>
+            <version>0.1.16</version>
         </dependency>
 
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-common</artifactId>
-            <version>0.0.10-SNAPSHOT</version>
+            <version>0.0.12</version>
         </dependency>
 
         <dependency>

provider/indexer-gcp/scripts/deploy2gcp.sh.tmpl

@@ -70,6 +70,7 @@ then
         sed -i -e "s|CRON_JOB_IP_VAR|$CRON_JOB_IP|g" app.yaml
         sed -i -e "s|LEGAL_HOSTNAME_VAR|$LEGAL_HOSTNAME|g" app.yaml
         sed -i -e "s|REGION_VAR|$REGION|g" app.yaml
+        sed -i -e "s|ENVIRONMENT|$ENVIRONMENT|g" app.yaml
 
         echo "Content of app.yaml:"
         cat app.yaml

provider/indexer-gcp/src/main/appengine/app.yaml

@@ -38,3 +38,4 @@ env_variables:
   CRON_JOB_IP: "CRON_JOB_IP_VAR"
   LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR"
   REGION: "REGION_VAR"
+  SPRING_PROFILES_ACTIVE: 'ENVIRONMENT'

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/AppengineLogFactory.java

+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.logging.ILogger;
+import org.opengroup.osdu.core.gcp.logging.logger.AppEngineLoggingProvider;
+
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Component
+@Primary
+@Lazy
+public class AppengineLogFactory implements FactoryBean<ILogger> {
+
+    private AppEngineLoggingProvider appEngineLoggingProvider = new AppEngineLoggingProvider();
+
+    @Override
+    public ILogger getObject() throws Exception {
+        return appEngineLoggingProvider.getLogger();
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return ILogger.class;
+    }
+}
\ No newline at end of file

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java

@@ -19,11 +19,13 @@ import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
 import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Component;
 import org.springframework.web.context.annotation.RequestScope;
 
 @Component
 @RequestScope
+@Lazy
 public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
 
 	@Value("${AUTHORIZE_API}")

provider/indexer-gcp/src/main/resources/application-dev.properties

+GOOGLE_CLOUD_PROJECT=opendes
+
+INDEXER_HOST=os-indexer-dot-opendes.appspot.com
+STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
+
+STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue
+
+AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
+LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
+CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
+
+## use below values for gcp: opendes
+REDIS_GROUP_HOST=10.0.16.28
+REDIS_SEARCH_HOST=10.0.16.20
+
+GOOGLE_AUDIENCES=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
\ No newline at end of file

provider/indexer-gcp/src/main/resources/application-testing.properties

+GOOGLE_CLOUD_PROJECT=opendes-evt
+
+INDEXER_HOST=os-indexer-dot-opendes-evt.appspot.com
+STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com
+
+STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-opendes-evt.appspot.com/_dps/task-handlers/enqueue
+
+AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1
+LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1
+CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1
+
+## use below values for gcp: opendes
+REDIS_GROUP_HOST=10.253.209.196
+REDIS_SEARCH_HOST=10.118.2.140
+
+GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com

provider/indexer-gcp/src/main/resources/application.properties

@@ -3,32 +3,15 @@ LOG_PREFIX=indexer
 server.servlet.contextPath=/api/indexer/v2
 logging.level.org.springframework.web=DEBUG
 server.port=8080
-#spring.cloud.gcp.project-id=opendes
-#spring.security.user.roles=service.search.admin
 JAVA_OPTS=-Xms3072m -Xmx3072m
 JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
 
-GOOGLE_AUDIENCES=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
-
-## use below values for gcp: local
-#REDIS_GROUP_HOST=127.0.0.1
-#REDIS_SEARCH_HOST=localhost
-#DEPLOYMENT_ENVIRONMENT=LOCAL
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=10.0.16.28
-REDIS_SEARCH_HOST=10.0.16.20
 DEPLOYMENT_ENVIRONMENT=CLOUD
 
 REDIS_GROUP_PORT=6379
 REDIS_SEARCH_PORT=6379
 DEFAULT_DATA_COUNTRY=US
 
-AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
-
-
 #Default Cache Settings
 SCHEMA_CACHE_EXPIRATION=60
 INDEX_CACHE_EXPIRATION=60
@@ -43,21 +26,9 @@ KINDS_REDIS_DATABASE=1
 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
 
-GOOGLE_CLOUD_PROJECT=opendes
 GAE_SERVICE=indexer
-
-INDEXER_HOST=os-indexer-dot-opendes.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
-
 KEY_RING=csqp
 KMS_KEY=searchService
 
 ELASTIC_DATASTORE_KIND=SearchSettings
 ELASTIC_DATASTORE_ID=indexer-service
-
-STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
-STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
-STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
-STORAGE_RECORDS_BATCH_SIZE=20
-
-INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue

testing/indexer-test-aws/pom.xml

@@ -44,7 +44,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>aws-osdu-util</artifactId>
-            <version>0.0.6-SNAPSHOT</version>
+            <version>0.0.8</version>
         </dependency>
 
         <!-- AWS managed packages -->

testing/indexer-test-azure/pom.xml

@@ -32,7 +32,19 @@
         <azure.keyvault.version>4.1.0</azure.keyvault.version>
         <junit.jupiter.version>5.6.0</junit.jupiter.version>
     </properties>
-
+    <!-- indexer test core depends on core libraries in OSDU, so the repository needs to be configured -->
+    <repositories>
+        <repository>
+            <id>os-core</id>
+            <url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
     <dependencies>
 
         <!-- Internal packages -->
@@ -46,7 +58,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-azure</artifactId>
-            <version>0.0.6-SNAPSHOT</version>
+            <version>0.0.8</version>
         </dependency>
 
         <dependency>

testing/indexer-test-core/pom.xml

@@ -14,11 +14,24 @@
         <cucumber.version>1.2.5</cucumber.version>
     </properties>
 
+    <!-- testing core depends on core libraries in OSDU, so the repository needs to be configured -->
+    <repositories>
+        <repository>
+            <id>os-core</id>
+            <url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
     <dependencies>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-common</artifactId>
-            <version>0.0.10-SNAPSHOT</version>
+            <version>0.0.12</version>
         </dependency>
 
         <dependency>
@@ -122,4 +135,4 @@
             <version>27.1-jre</version>
         </dependency>
     </dependencies>
-</project>
\ No newline at end of file
+</project>