Merge branch 'master' into retry

provider/indexer-gc/docs/gc/README.md

@@ -201,7 +201,7 @@ You will need to have the following environment variables defined.
 | `LEGAL_TAG`                         | ex `opendes-demo-legaltag`                                            | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no         | -                                                            |
 | `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes`                                                          | HTTP Header 'Data-Partition-ID'                                                                   | no         | -                                                            |
 | `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes`                                                          | HTTP Header 'Data-Partition-ID'                                                                   | no         | -                                                            |
-| `SEARCH_INTEGRATION_TESTER`         | `********`                                                            | Service account for API calls. Note: this user must have entitlements configured already          | yes        | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| `SEARCH_INTEGRATION_TESTER`         | `ewogICJ0....` or `tmp/service-acc.json`                              | Service account for API calls. Note: this user must have entitlements configured already          | yes        | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
 | `SEARCH_HOST`                       | ex `http://localhost:8080/api/search/v2/`                             | Endpoint of search service                                                                        | no         | -                                                            |
 | `STORAGE_HOST`                      | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/`        | Storage API endpoint                                                                              | no         | output of infrastructure deployment                          |
 | `SECURITY_HTTPS_CERTIFICATE_TRUST`  | ex `false`                                                            | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true'                         | false      | output of infrastructure deployment                          |

provider/indexer-gc/src/main/resources/application.properties

@@ -69,7 +69,6 @@ featureFlag.xCollaboration.enabled=false
 featureFlag.asIngestedCoordinates.enabled=true
 featureFlag.keywordLower.enabled=true
 featureFlag.bagOfWords.enabled=true
-featureFlag.xCollaboration.enabled=false
 
 # Health checks
 management.server.port=${MANAGEMENT_SERVER_PORT:8081}

testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/DecodedContentExtractor.java

+/*
+ *  Copyright 2020-2024 Google LLC
+ *  Copyright 2020-2024 EPAM Systems, Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.opengroup.osdu.util;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.function.Predicate;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.java.Log;
+
+@Log
+@RequiredArgsConstructor
+public class DecodedContentExtractor {
+    private final String inputFilenameOrContent;
+    private final Predicate<String> contentAcceptanceTester;
+
+    private boolean validOutputContentFound;
+    private String outputContent;
+
+    public String getContent() {
+
+        validOutputContentFound = false;
+        outputContent = null;
+
+        log.info("Treat value as a content");
+        if (inputFilenameOrContent.trim().isEmpty()) {
+            log.info("provided value is empty. Output as is.");
+            return setValidOutputContent(inputFilenameOrContent);
+        }
+
+        if (!treatValueAsAContent(inputFilenameOrContent)) {
+            log.info("Value is not a valid content. Treat value as a filename");
+            if (!treatValueAsAFileName(inputFilenameOrContent)){
+                log.info("Value is not a filename with a valid content");
+            }
+
+        }
+
+        return getValidOutputContentIfFound();
+    }
+
+    private boolean treatValueAsAContent(String input) {
+        if (contentAcceptanceTester.test(input)) {
+            log.info("the value is a valid content. Output as is.");
+            setValidOutputContent(input);
+            return true;
+        }
+        String output;
+        try {
+            output = new String(Base64.getDecoder().decode(input));
+            log.info("the value is probably Base64 encoded. Just decoded");
+            if (contentAcceptanceTester.test(output)) {
+                log.info("the decoded value is a valid content. Output decoded value.");
+                setValidOutputContent(output);
+            } else {
+                log.info("the decoded value is not a valid content.");
+            }
+        } catch (IllegalArgumentException e) {
+            log.info("the value is not Base64 encoded. ");
+        }
+
+        return validOutputContentFound;
+    }
+
+    private boolean treatValueAsAFileName(String filename) {
+
+        if (treatFileContent(filename)) return true;
+
+        try {
+            filename = new String(Base64.getDecoder().decode(filename));
+            log.info("the filename is probably Base64 encoded. Just decoded");
+            if (treatFileContent(filename)) return true;
+        } catch (IllegalArgumentException e) {
+            log.info("the filename is not Base64 encoded. ");
+        }
+        return validOutputContentFound;
+    }
+
+    private boolean treatFileContent(String filename) {
+        try {
+            Path path = Paths.get(filename);
+            if (Files.exists(path)) {
+                log.info("the filename is of existing file. Read file.");
+                try {
+                    String fileContent = new String(Files.readAllBytes(path));
+                    if (treatValueAsAContent(fileContent)) {
+                        return true;
+                    }
+                } catch (IOException | SecurityException | OutOfMemoryError ex) {
+                    log.info(() -> ("unable to read the file: " + ex.getClass().getSimpleName()));
+                }
+            }
+        } catch (InvalidPathException ex) {
+            log.info("the filename is not valid or the file doesn't exist.");
+        }
+        return false;
+    }
+
+    private String setValidOutputContent(String outputContent) {
+        this.outputContent = outputContent;
+        this.validOutputContentFound = true;
+        return getValidOutputContentIfFound();
+    }
+
+    public String getValidOutputContentIfFound() {
+        return validOutputContentFound ? outputContent : null;
+    }
+}

testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java

@@ -8,6 +8,13 @@ import com.google.api.client.json.webtoken.JsonWebToken;
 import com.google.api.client.util.Clock;
 import com.google.common.base.Strings;
 import com.google.gson.Gson;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
 import lombok.Data;
 import org.apache.commons.io.Charsets;
 import org.apache.commons.io.IOUtils;
@@ -19,20 +26,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicNameValuePair;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.List;
-
 class JwtTokenUtil {
 
+    private final static Predicate<String> contentAcceptanceTester = s -> s.trim().startsWith("{");
     private static String accessToken;
 
     static String getAccessToken() throws IOException {
-
         if (Strings.isNullOrEmpty(accessToken)) {
             accessToken = getServiceAccountAccessToken(getJwtForIntegrationTesterAccount());
         }
@@ -61,14 +60,14 @@ class JwtTokenUtil {
     }
 
     private static String getJwtForIntegrationTesterAccount() throws IOException {
-        String serviceAccountFile = Config.getKeyValue();
-        return getJwt(serviceAccountFile);
+        String serviceAccountValue = new DecodedContentExtractor(Config.getKeyValue(), contentAcceptanceTester).getContent();
+        return getJwt(serviceAccountValue);
     }
 
     private static String getJwt(String serviceAccountFile) throws IOException {
         long currentTime = Clock.SYSTEM.currentTimeMillis();
 
-        InputStream stream = new ByteArrayInputStream(Base64.getDecoder().decode(serviceAccountFile));
+        InputStream stream = new ByteArrayInputStream(serviceAccountFile.getBytes());
         GoogleCredential credential = GoogleCredential.fromStream(stream);
 
         JsonWebSignature.Header header = new JsonWebSignature.Header();