GONRG-1041 Indexer should support HTTP Target Cloud Tasks

Changes:
*Added request authentication incoming from Http cloud task
based on token verification
*Updated core-lib-gcp version in pom
*Added property with service acc mail, for token validation
*Updated google-cloud-datastore lib version , to reuse it's Oauth2 dependencies

.gitlab-ci.yml

@@ -14,13 +14,14 @@ variables:
   GCP_DOMAIN: cloud.slb-ds.com
   GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
 
-  OSDU_GCP_BUILD_SUBDIR: provider/indexer-gcp
-  OSDU_GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
   OSDU_GCP_APPLICATION_NAME: os-indexer
-  OSDU_GCP_PROJECT: nice-etching-277309
-  OSDU_GCP_TENANT_NAME: osdu
-  OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+  OSDU_GCP_SERVICE: indexer
+  OSDU_GCP_VENDOR: gcp
+  OSDU_GCP_SERVICE_ACCOUNT: osdu-gcp-sa@nice-etching-277309.iam.gserviceaccount.com
   OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
+  OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE: 20
+  OSDU_GCP_DATA_GROUP: osdu
+  OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCES,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_SERVICE_ACCOUNT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
 
   IBM_BUILD_SUBDIR: provider/indexer-ibm
   IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm
@@ -42,22 +43,22 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/gitlab-ultimate.yml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/aws.yml"
+  # - project: "osdu/platform/ci-cd-pipelines"
+  #   file: "cloud-providers/aws.yml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/azure.yml"
+  # - project: "osdu/platform/ci-cd-pipelines"
+  #   file: "cloud-providers/azure.yml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/ibm.yml"
+  # - project: "osdu/platform/ci-cd-pipelines"
+  #   file: "cloud-providers/ibm.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "publishing/pages.yml"
 
   - project: 'osdu/platform/ci-cd-pipelines'
-    ref: "master"
-    file: 'cloud-providers/osdu-gcp.yml'
+    ref: "osdu-gcp-add-vars-for-indexer"
+    file: 'cloud-providers/osdu-gcp-cloudrun.yml'
 
 
-aws-test-java:
-  tags: ['aws-internal-test']
+# aws-test-java:
+  # tags: ['aws-internal-test']

provider/indexer-gcp/README.md

@@ -35,7 +35,7 @@ In order to run the service locally or remotely, you will need to have the follo
 | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
 | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
 | `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-
+| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | IndexerQue environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
 
 ### Run Locally
 Check that maven is installed:

provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild

+# Use the official AdoptOpenJDK for a base image.
+# https://hub.docker.com/_/openjdk
+FROM openjdk:8-slim
+WORKDIR /app
+ARG PROVIDER_NAME
+ENV PROVIDER_NAME $PROVIDER_NAME
+ARG PORT
+ENV PORT $PORT
+# Copy the jar to the production image from the builder stage.
+COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-SNAPSHOT-spring-boot.jar indexer-${PROVIDER_NAME}.jar
+# Run the web service on container startup.
+CMD java -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -jar /app/indexer-${PROVIDER_NAME}.jar
+

provider/indexer-gcp/cloudbuild/cloudbuild.yaml

+# Copyright 2020 Google LLC
+# Copyright 2017-2019, Schlumberger
+# Copyright 2020 EPAM
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: [
+            'build',
+            '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
+            '--build-arg', 'PORT=${_PORT}',
+            '-t', 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}:${_SHORT_SHA}',
+            '-t', 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}:latest',
+            '-f', 'provider/indexer-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild',
+            '.'
+    ]
+
+images:
+  - 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}'
+

provider/indexer-gcp/pom.xml

@@ -27,7 +27,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-gcp</artifactId>
-            <version>0.3.21</version>
+            <version>0.3.23</version>
         </dependency>
 
         <dependency>
@@ -67,6 +67,11 @@
             <artifactId>google-cloud-pubsub</artifactId>
             <version>1.60.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.api-client</groupId>
+            <artifactId>google-api-client</artifactId>
+            <version>1.30.11</version>
+        </dependency>
 
         <dependency>
             <groupId>org.elasticsearch</groupId>

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java

@@ -24,7 +24,7 @@ import javax.inject.Inject;
 import com.google.common.base.Strings;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 
-import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
 import org.opengroup.osdu.core.gcp.util.TraceIdExtractor;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
@@ -43,10 +43,10 @@ public class DpsHeaderFactoryGcp extends DpsHeaders {
                 .stream()
                 .collect(Collectors.toMap(h -> h, request::getHeader));
 
-        String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+        String traceContext = headers.get(CloudTaskHeaders.CLOUD_TRACE_CONTEXT);
 
         if(!Strings.isNullOrEmpty(traceContext)){
-            headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
+            headers.put(CloudTaskHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
         }
 
         this.addFromMap(headers);

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java

 package org.opengroup.osdu.indexer.util;
 
+import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken;
+import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.common.base.Strings;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.logging.Level;
 import lombok.extern.java.Log;
 import org.apache.http.HttpStatus;
 import org.opengroup.osdu.core.common.Constants;
@@ -9,8 +17,8 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
 import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
 import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.web.context.annotation.RequestScope;
@@ -38,6 +46,9 @@ public class RequestInfoImpl implements IRequestInfo {
     @Value("${DEPLOYMENT_ENVIRONMENT}")
     private String DEPLOYMENT_ENVIRONMENT;
 
+    @Value("${indexer.que.service.mail}")
+    private String indexerQueServiceMail;
+
     private static final String expectedCronHeaderValue = "true";
 
     @Override
@@ -70,15 +81,53 @@ public class RequestInfoImpl implements IRequestInfo {
 
     @Override
     public boolean isCronRequest() {
-        String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
+        String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null);
         return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
     }
 
     @Override
     public boolean isTaskQueueRequest() {
-        if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false;
+        if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){
+            log.log(Level.INFO,"Request acknowledged as Cloud task, proceeding token validation");
+            return isCloudTaskRequest();
+        }
+        if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){
+            log.log(Level.INFO,"Request acknowledged as AppEngine task, proceeding headers validation");
+            return isAppEngineTaskRequest();
+        }
+        return false;
+    }
+
+    private boolean isCloudTaskRequest() {
+        log.log(Level.INFO,dpsHeaders.getHeaders().toString());
+        try {
+            GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance())
+                .setIssuers(Arrays.asList(
+                    "accounts.google.com", "https://accounts.google.com",
+                    "googleapis.com", "https://www.googleapis.com/auth/userinfo.profile"
+                    )
+                ).build();
+            String authorization = dpsHeaders.getAuthorization().replace("Bearer ", "");
+
+            GoogleIdToken googleIdToken = verifier.verify(authorization);
+            if(googleIdToken.getPayload().getEmail().equals(indexerQueServiceMail)){
+                return true;
+            }
+            log.log(Level.WARNING,"Token email doesn't match with variable \"indexer.que.service.mail\"");
+            return false;
 
-        String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME);
+        } catch (GeneralSecurityException | IOException e) {
+            log.log(Level.WARNING,"Not valid or expired cloud task token provided");
+            return false;
+        }
+    }
+
+    private boolean isAppEngineTaskRequest(){
+        if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
+            return false;
+        }
+        String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME);
         return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
     }
 

provider/indexer-gcp/src/main/resources/application.properties

@@ -34,3 +34,4 @@ ELASTIC_DATASTORE_KIND=SearchSettings
 ELASTIC_DATASTORE_ID=indexer-service
 
 security.https.certificate.trust=false
+indexer.que.service.mail=default@iam.gserviceaccount.com