diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f03a7d4989795cac0acbcbbb43dd28911d1e05bd..46503d4175c9ea0920459e8f299edccdf4dbc52f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,13 +14,14 @@ variables: GCP_DOMAIN: cloud.slb-ds.com GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/ - OSDU_GCP_BUILD_SUBDIR: provider/indexer-gcp - OSDU_GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp OSDU_GCP_APPLICATION_NAME: os-indexer - OSDU_GCP_PROJECT: nice-etching-277309 - OSDU_GCP_TENANT_NAME: osdu - OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas + OSDU_GCP_SERVICE: indexer + OSDU_GCP_VENDOR: gcp + OSDU_GCP_SERVICE_ACCOUNT: osdu-gcp-sa@nice-etching-277309.iam.gserviceaccount.com OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true' + OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE: 20 + OSDU_GCP_DATA_GROUP: osdu + OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCES,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_SERVICE_ACCOUNT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR IBM_BUILD_SUBDIR: provider/indexer-ibm IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm @@ -42,22 +43,22 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "scanners/gitlab-ultimate.yml" - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/aws.yml" + # - project: "osdu/platform/ci-cd-pipelines" + # file: "cloud-providers/aws.yml" - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/azure.yml" + # - project: "osdu/platform/ci-cd-pipelines" + # file: "cloud-providers/azure.yml" - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/ibm.yml" + # - project: "osdu/platform/ci-cd-pipelines" + # file: "cloud-providers/ibm.yml" - project: "osdu/platform/ci-cd-pipelines" file: "publishing/pages.yml" - project: 'osdu/platform/ci-cd-pipelines' - ref: "master" - file: 'cloud-providers/osdu-gcp.yml' + ref: "osdu-gcp-add-vars-for-indexer" + file: 'cloud-providers/osdu-gcp-cloudrun.yml' -aws-test-java: - tags: ['aws-internal-test'] +# aws-test-java: + # tags: ['aws-internal-test'] diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index 23c482574d2515459a8f962b197687472e1bba52..373972ee9e4561bab282bb0e84e80954293724f0 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -35,7 +35,7 @@ In order to run the service locally or remotely, you will need to have the follo | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | | `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | - +| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | IndexerQue environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | ### Run Locally Check that maven is installed: diff --git a/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild b/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild new file mode 100644 index 0000000000000000000000000000000000000000..59c9b5dafcf6824308be6cf80357a887ac75bd36 --- /dev/null +++ b/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild @@ -0,0 +1,13 @@ +# Use the official AdoptOpenJDK for a base image. +# https://hub.docker.com/_/openjdk +FROM openjdk:8-slim +WORKDIR /app +ARG PROVIDER_NAME +ENV PROVIDER_NAME $PROVIDER_NAME +ARG PORT +ENV PORT $PORT +# Copy the jar to the production image from the builder stage. +COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-SNAPSHOT-spring-boot.jar indexer-${PROVIDER_NAME}.jar +# Run the web service on container startup. +CMD java -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -jar /app/indexer-${PROVIDER_NAME}.jar + diff --git a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f3b4e18377dffb2635aa65c5cbe5cdc7b2b9a68 --- /dev/null +++ b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml @@ -0,0 +1,31 @@ +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'gcr.io/cloud-builders/docker' + args: [ + 'build', + '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', + '--build-arg', 'PORT=${_PORT}', + '-t', 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}:${_SHORT_SHA}', + '-t', 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}:latest', + '-f', 'provider/indexer-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', + '.' + ] + +images: + - 'gcr.io/$PROJECT_ID/os-indexer/indexer-${_PROVIDER_NAME}' + diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml index 70abe1c66847b343a9ae5b730163f50120b32d35..2b91fa1322e2b54eb010e91ef1af984f70de7ed7 100644 --- a/provider/indexer-gcp/pom.xml +++ b/provider/indexer-gcp/pom.xml @@ -27,7 +27,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-gcp</artifactId> - <version>0.3.21</version> + <version>0.3.23</version> </dependency> <dependency> @@ -67,6 +67,11 @@ <artifactId>google-cloud-pubsub</artifactId> <version>1.60.0</version> </dependency> + <dependency> + <groupId>com.google.api-client</groupId> + <artifactId>google-api-client</artifactId> + <version>1.30.11</version> + </dependency> <dependency> <groupId>org.elasticsearch</groupId> diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java index 28884453076f6ee465f876c9a014bdeda0da935b..75b7b744f9e26ce7fb58315ccb14c28b34eef42a 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java @@ -24,7 +24,7 @@ import javax.inject.Inject; import com.google.common.base.Strings; import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; +import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; import org.opengroup.osdu.core.gcp.util.TraceIdExtractor; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -43,10 +43,10 @@ public class DpsHeaderFactoryGcp extends DpsHeaders { .stream() .collect(Collectors.toMap(h -> h, request::getHeader)); - String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); + String traceContext = headers.get(CloudTaskHeaders.CLOUD_TRACE_CONTEXT); if(!Strings.isNullOrEmpty(traceContext)){ - headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); + headers.put(CloudTaskHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); } this.addFromMap(headers); diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 2ad14667d9aa2e47876989d98ab124c79cc2c618..8ab213d952a647db23ae61ab892fd12fbdb72d25 100644 --- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -1,6 +1,14 @@ package org.opengroup.osdu.indexer.util; +import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken; +import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; import com.google.common.base.Strings; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.logging.Level; import lombok.extern.java.Log; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; @@ -9,8 +17,8 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -38,6 +46,9 @@ public class RequestInfoImpl implements IRequestInfo { @Value("${DEPLOYMENT_ENVIRONMENT}") private String DEPLOYMENT_ENVIRONMENT; + @Value("${indexer.que.service.mail}") + private String indexerQueServiceMail; + private static final String expectedCronHeaderValue = "true"; @Override @@ -70,15 +81,53 @@ public class RequestInfoImpl implements IRequestInfo { @Override public boolean isCronRequest() { - String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null); + String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null); return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); } @Override public boolean isTaskQueueRequest() { - if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false; + if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){ + log.log(Level.INFO,"Request acknowledged as Cloud task, proceeding token validation"); + return isCloudTaskRequest(); + } + if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){ + log.log(Level.INFO,"Request acknowledged as AppEngine task, proceeding headers validation"); + return isAppEngineTaskRequest(); + } + return false; + } + + private boolean isCloudTaskRequest() { + log.log(Level.INFO,dpsHeaders.getHeaders().toString()); + try { + GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder( + GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance()) + .setIssuers(Arrays.asList( + "accounts.google.com", "https://accounts.google.com", + "googleapis.com", "https://www.googleapis.com/auth/userinfo.profile" + ) + ).build(); + String authorization = dpsHeaders.getAuthorization().replace("Bearer ", ""); + + GoogleIdToken googleIdToken = verifier.verify(authorization); + if(googleIdToken.getPayload().getEmail().equals(indexerQueServiceMail)){ + return true; + } + log.log(Level.WARNING,"Token email doesn't match with variable \"indexer.que.service.mail\""); + return false; - String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); + } catch (GeneralSecurityException | IOException e) { + log.log(Level.WARNING,"Not valid or expired cloud task token provided"); + return false; + } + } + + private boolean isAppEngineTaskRequest(){ + if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) { + return false; + } + String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME); return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); } diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index a0cbce3427ff62ba4d63ffabc17323efc833295b..e4391d77b133e71e5f52b8f39753a746fd85b5d6 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -34,3 +34,4 @@ ELASTIC_DATASTORE_KIND=SearchSettings ELASTIC_DATASTORE_ID=indexer-service security.https.certificate.trust=false +indexer.que.service.mail=default@iam.gserviceaccount.com