Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/search-service

provider/search-aws/build-aws/buildspec.yaml

@@ -77,7 +77,7 @@ phases:
       - if [ "$GIT_SECRETS_SCAN_RESULT" = "FAILED" ]; then echo "Secrets detected!" && exit 1; fi
 
       - echo "Building primary service assemblies..."
-      - mvn -ntp -B test install sonar:sonar -pl .,search-core,provider/search-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} 
+      - mvn -ntp -B test install sonar:sonar -pl .,search-core,provider/search-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
 
       - echo "Building integration testing assemblies and gathering artifacts..."
       - ./testing/integration-tests/search-test-aws/build-aws/prepare-dist.sh

provider/search-aws/src/main/java/org/opengroup/osdu/search/provider/aws/provider/impl/QueryBase.java

@@ -62,6 +62,7 @@ import org.opengroup.osdu.core.common.model.search.QueryUtils;
 import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
 import org.opengroup.osdu.core.common.model.search.SpatialFilter;
 import org.opengroup.osdu.search.policy.service.IPolicyService;
+import org.opengroup.osdu.search.provider.aws.util.AwsCrossTenantUtils;
 import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
 import org.opengroup.osdu.search.service.IFieldMappingTypeService;
 import org.opengroup.osdu.search.util.AggregationParserUtil;
@@ -84,7 +85,7 @@ abstract class QueryBase {
     @Inject
     private IProviderHeaderService providerHeaderService;
     @Inject
-    private CrossTenantUtils crossTenantUtils;
+    private AwsCrossTenantUtils crossTenantUtils;
     @Inject
     private IFieldMappingTypeService fieldMappingTypeService;
     @Autowired(required = false)

provider/search-aws/src/main/java/org/opengroup/osdu/search/provider/aws/util/AwsCrossTenantUtils.java

+package org.opengroup.osdu.search.provider.aws.util;
+
+import org.opengroup.osdu.core.common.exception.BadRequestException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.search.util.CrossTenantUtils;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.model.search.Query;
+import org.opengroup.osdu.core.common.util.KindParser;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.List;
+
+@Component
+public class AwsCrossTenantUtils extends CrossTenantUtils {
+
+    @Inject
+    private DpsHeaders dpsHeaders;
+    @Inject
+    private ElasticIndexNameResolver elasticIndexNameResolver;
+
+    // This override method forces the kind used in the index search to use the same partition id as
+    // the request's partition id header. This is to prevent data from other tenants (aka. partition)
+    // to appear when searching data in a given tenant.
+    @Override
+    public String getIndexName(Query searchRequest) throws BadRequestException {
+        StringBuilder builder = new StringBuilder();
+        List<String> kinds = KindParser.parse(searchRequest.getKind());
+        for(String kind : kinds) {
+            String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+            String[] indexArr = index.split("-");
+            if (indexArr[0].equalsIgnoreCase("*")) {
+                indexArr[0] = dpsHeaders.getPartitionId();
+            }
+            else if (indexArr[0].equalsIgnoreCase(dpsHeaders.getPartitionId()) == false) {
+                throw new BadRequestException("Invalid kind in search request.");
+            }
+            builder.append(String.join("-", indexArr) + ",");
+        }
+        builder.append("-.*"); // Exclude Lucene/ElasticSearch internal indices
+        return builder.toString();
+    }
+
+}

provider/search-aws/src/test/java/org/opengroup/osdu/search/provider/aws/provider/impl/QueryServiceAwsImplTest.java

@@ -36,6 +36,7 @@ import org.opengroup.osdu.core.common.model.search.*;
 import org.opengroup.osdu.search.logging.AuditLogger;
 import org.opengroup.osdu.search.policy.service.PartitionPolicyStatusService;
 import org.opengroup.osdu.search.provider.aws.provider.impl.QueryServiceAwsImpl;
+import org.opengroup.osdu.search.provider.aws.util.AwsCrossTenantUtils;
 import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
 import org.opengroup.osdu.search.service.IFieldMappingTypeService;
 import org.opengroup.osdu.search.util.*;
@@ -68,7 +69,7 @@ public class QueryServiceAwsImplTest {
 	private IProviderHeaderService providerHeaderService;
 
 	@Mock
-	private CrossTenantUtils crossTenantUtils;
+	private AwsCrossTenantUtils crossTenantUtils;
 
 	@Mock
 	private IFieldMappingTypeService fieldMappingTypeService;

provider/search-aws/src/test/java/org/opengroup/osdu/search/provider/aws/util/AwsCrossTenantUtilsTest.java

+package org.opengroup.osdu.search.provider.aws.util;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.*;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.exception.BadRequestException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.*;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AwsCrossTenantUtilsTest {
+
+    @InjectMocks
+    AwsCrossTenantUtils awsCrossTenantUtils;
+
+    @Mock
+    private DpsHeaders dpsHeaders;
+
+    @Mock
+    private ElasticIndexNameResolver elasticIndexNameResolver;
+
+    @Mock
+    private Query searchRequest;
+
+    @Test
+    public void should_return_IndexName_when_partitionIdAndKindMatches() {
+        String DATA_PARTITION_ID = "tenant1";
+        String KIND = "tenant1:*:*:*";
+        String INDEX = KIND.replace(":", "-");
+        String INDEX_NAME = String.format("%s,%s", INDEX, "-.*");
+        when(dpsHeaders.getPartitionId()).thenReturn(DATA_PARTITION_ID);
+        when(searchRequest.getKind()).thenReturn(KIND);
+        when(elasticIndexNameResolver.getIndexNameFromKind(KIND)).thenReturn(INDEX);
+
+        assertEquals(INDEX_NAME, awsCrossTenantUtils.getIndexName(searchRequest));
+    }
+
+    @Test
+    public void should_return_PartitionIdIndexName_when_searchingAllKinds() {
+        String DATA_PARTITION_ID = "tenant1";
+        String KIND = "*:*:*:*";
+        String INDEX = KIND.replace(":", "-");
+        String INDEX_NAME = String.format("%s%s,%s", DATA_PARTITION_ID, "-*-*-*", "-.*");
+        when(dpsHeaders.getPartitionId()).thenReturn(DATA_PARTITION_ID);
+        when(searchRequest.getKind()).thenReturn(KIND);
+        when(elasticIndexNameResolver.getIndexNameFromKind(KIND)).thenReturn(INDEX);
+
+        assertEquals(INDEX_NAME, awsCrossTenantUtils.getIndexName(searchRequest));
+    }
+
+    @Test(expected = BadRequestException.class)
+    public void should_throw_BadRequestException_when_searchingAcrossTenants() {
+        String DATA_PARTITION_ID = "tenant1";
+        String KIND = "tenant2:*:*:*";
+        String INDEX = KIND.replace(":", "-");
+        String INDEX_NAME = String.format("%s,%s", INDEX, "-.*");
+        when(dpsHeaders.getPartitionId()).thenReturn(DATA_PARTITION_ID);
+        when(searchRequest.getKind()).thenReturn(KIND);
+        when(elasticIndexNameResolver.getIndexNameFromKind(KIND)).thenReturn(INDEX);
+
+        awsCrossTenantUtils.getIndexName(searchRequest); // Should throw an exception.
+    }
+}

provider/search-azure/pom.xml

@@ -56,6 +56,7 @@
         <azure-core.version>1.31.0</azure-core.version>
         <azure-security-keyvault-keys.version>4.4.6</azure-security-keyvault-keys.version>
         <azure-security-keyvault-secrets.version>4.4.6</azure-security-keyvault-secrets.version>
+        <osdu.oscorecommon.version>0.19.0-rc6</osdu.oscorecommon.version>
     </properties>
 
     <dependencyManagement>

provider/search-azure/src/main/java/org/opengroup/osdu/search/provider/azure/provider/impl/AzureAuthorizationService.java

+
+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.search.provider.azure.provider.impl;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse;
+import org.opengroup.osdu.core.common.model.entitlements.Groups;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
+import org.opengroup.osdu.search.service.IEntitlementsExtensionService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+
+@Service
+@Primary // overrides class in core common
+public class AzureAuthorizationService implements IAuthorizationService {
+
+    @Autowired
+    private IEntitlementsExtensionService entitlementsService;
+
+    @Override
+    public AuthorizationResponse authorizeAny(DpsHeaders dpsHeaders, String... permissions) {
+        Groups groups = entitlementsService.getGroups(dpsHeaders);
+        if(!groups.any(permissions)) {
+            throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Unauthorized", "User does nto have access to the API");
+        }
+        return AuthorizationResponse.builder()
+                .user(groups.getMemberEmail())
+                .groups(groups)
+                .build();
+    }
+
+    @Override
+    public AuthorizationResponse authorizeAny(String partition, DpsHeaders dpsHeaders, String... permissions) {
+        throw new NotImplementedException("authorizeAny not implemented for azure");
+    }
+}
\ No newline at end of file

provider/search-azure/src/main/resources/application.properties

@@ -89,3 +89,5 @@ service.policy.enabled=true
 service.policy.endpoint=${policy_service_endpoint}
 policy.cache.timeout=5
 service.policy.id=${service_policy_id:osdu.partition["%s"].search}
+
+validation.spatial.longitude.enableExtendedRange=true

provider/search-azure/src/test/java/org/opengroup/osdu/search/provider/azure/provider.impl/AzureAuthorizationServiceTest.java

+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.search.provider.azure.provider.impl;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.entitlements.Groups;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.search.service.IEntitlementsExtensionService;
+
+import javax.ws.rs.NotSupportedException;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AzureAuthorizationServiceTest {
+
+    @Mock
+    private IEntitlementsExtensionService entitlementsService;
+
+    @Mock
+    private Groups groups;
+
+    @InjectMocks
+    private AzureAuthorizationService sut;
+
+    @Test(expected = AppException.class)
+    public void should_throwAppException_when_givenGroupDoesNotExistForUser() {
+        when(groups.any(any())).thenReturn(false);
+        when(entitlementsService.getGroups(any())).thenReturn(groups);
+        sut.authorizeAny(new DpsHeaders(), "a", "b");
+    }
+
+    @Test
+    public void should_returnGroupsWithUserEmail_when_givenGroupExistsForUser() {
+        when(groups.any(any())).thenReturn(true);
+        when(entitlementsService.getGroups(any())).thenReturn(groups);
+        sut.authorizeAny(new DpsHeaders(), "a", "b");
+    }
+}

testing/integration-tests/search-test-azure/src/test/java/org/opengroup/osdu/step_definitions/query/singlecluster/RunTest.java

@@ -21,7 +21,7 @@ import org.junit.runner.RunWith;
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-        features = "classpath:features/query/singlecluster/Query.feature",
+        features = "classpath:features/query/singlecluster/SingleClusterQuery.feature",
         glue = {"classpath:org.opengroup.osdu.step_definitions/query/singlecluster"},
         plugin = {"pretty", "junit:target/cucumber-reports/TEST-query-sc.xml"})
 public class RunTest {

testing/integration-tests/search-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java

@@ -14,9 +14,16 @@
 
 package org.opengroup.osdu.util;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
 import lombok.ToString;
 import lombok.extern.java.Log;
 
+import javax.ws.rs.core.Response;
+import java.net.SocketTimeoutException;
+import java.util.Map;
+
 @Log
 @ToString
 public class AzureHTTPClient extends HTTPClient {
@@ -34,4 +41,44 @@ public class AzureHTTPClient extends HTTPClient {
         }
         return token;
     }
-}
\ No newline at end of file
+
+    public ClientResponse send(String httpMethod, String url, String payLoad, Map<String, String> headers, String token) {
+        ClientResponse response;
+        System.out.println("in Azure send method");
+        String correlationId = java.util.UUID.randomUUID().toString();
+        log.info(String.format("Request correlation id: %s", correlationId));
+        headers.put(HEADER_CORRELATION_ID, correlationId);
+        Client client = getClient();
+        client.setReadTimeout(300000);
+        client.setConnectTimeout(300000);
+        log.info(String.format("httpMethod: %s", httpMethod));
+        log.info(String.format("payLoad: %s", payLoad));
+        log.info(String.format("headers: %s", headers));
+        log.info(String.format("URL: %s", url));
+        WebResource webResource = client.resource(url);
+        log.info("waiting on response in azure send");
+        int retryCount = 2;
+        try{
+            response = this.getClientResponse(httpMethod, payLoad, webResource, headers, token);
+            while (retryCount > 0) {
+                if (response.getStatusInfo().getFamily().equals(Response.Status.Family.valueOf("SERVER_ERROR"))) {
+                    log.info(String.format("got resoponse : %s", response.getStatusInfo()));
+                    Thread.sleep(5000);
+                    log.info(String.format("Retrying --- "));
+                    response = this.getClientResponse(httpMethod, payLoad, webResource, headers, token);
+                } else
+                    break;
+                retryCount--;
+            }
+            System.out.println("sending response from azure send method");
+            return response;
+        } catch (Exception e) {
+            if (e.getCause() instanceof SocketTimeoutException) {
+                System.out.println("Retrying in case of socket timeout exception");
+                return this.getClientResponse(httpMethod, payLoad, webResource, headers, token);
+            }
+            e.printStackTrace();
+            throw new AssertionError("Error: Send request error", e);
+        }
+    }
+}

testing/integration-tests/search-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java

@@ -24,11 +24,11 @@ public abstract class HTTPClient {
     private static Random random = new Random();
     private final int MAX_ID_SIZE = 50;
 
-    private static final String HEADER_CORRELATION_ID = "correlation-id";
+    protected static final String HEADER_CORRELATION_ID = "correlation-id";
     
     public abstract String getAccessToken();
 
-    private static Client getClient() {
+    protected static Client getClient() {
         TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
             @Override
             public X509Certificate[] getAcceptedIssuers() {
@@ -60,7 +60,7 @@ public abstract class HTTPClient {
             log.info(String.format("Request correlation id: %s", correlationId));
             headers.put(HEADER_CORRELATION_ID, correlationId);
             Client client = getClient();
-            client.setReadTimeout(180000);
+            client.setReadTimeout(300000);
             client.setConnectTimeout(300000);
             log.info(String.format("URL: %s", url));
             WebResource webResource = client.resource(url);
@@ -73,7 +73,7 @@ public abstract class HTTPClient {
         return response;
     }
 
-    private ClientResponse getClientResponse(String httpMethod, String requestBody, WebResource webResource, Map<String, String> headers, String token) {
+    protected ClientResponse getClientResponse(String httpMethod, String requestBody, WebResource webResource, Map<String, String> headers, String token) {
         final WebResource.Builder builder = webResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).header("Authorization", token);
         headers.forEach(builder::header);
         log.info("making request to datalake api");