Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/indexer-service into muskans-retry-indexer

.gitlab-ci.yml

@@ -22,7 +22,7 @@ variables:
   OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
   OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE: 20
   OSDU_GCP_DATA_GROUP: osdu
-  OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_QUEUE_SA_EMAIL,SCHEMA_HOST=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1/schema --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
+  OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_QUEUE_SA_EMAIL,SCHEMA_HOST=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1/schema --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
   OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
 
   IBM_BUILD_SUBDIR: provider/indexer-ibm

NOTICE

@@ -209,6 +209,7 @@ The following software have components provided under the terms of this license:
 - Asynchronous Http Client (from )
 - Asynchronous Http Client Netty Utils (from )
 - AutoValue (from )
+- AutoValue Annotations (from )
 - Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
 - Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
 - BSON (from http://bsonspec.org)
@@ -237,6 +238,7 @@ The following software have components provided under the terms of this license:
 - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
 - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
 - FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- GSON extensions to the Google HTTP Client Library for Java. (from )
 - Google APIs Client Library for Java (from )
 - Google APIs Client Library for Java (from )
 - Google App Engine extensions to the Google HTTP Client Library for Java. (from )
@@ -381,9 +383,9 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
 - Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
 - Mockito (from http://mockito.org)
+- Mockito (from http://www.mockito.org)
 - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
 - MongoDB Driver (from http://www.mongodb.org)
 - MongoDB Java Driver Core (from http://www.mongodb.org)
@@ -634,6 +636,8 @@ The following software have components provided under the terms of this license:
 - Google APIs Client Library for Java (from )
 - Google APIs Client Library for Java (from )
 - Google Auth Library for Java - Credentials (from )
+- Google Auth Library for Java - Credentials (from )
+- Google Auth Library for Java - OAuth2 HTTP (from )
 - Google Auth Library for Java - OAuth2 HTTP (from )
 - Hamcrest library (from )
 - JDOM (from http://www.jdom.org)
@@ -1107,3 +1111,5 @@ The following software have components provided under the terms of this license:
 - jts-core (from )
 - jts-io-common (from )
 - xml-apis (from )
+
+

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java

@@ -244,7 +244,11 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
         HashMap<String, Object> propertiesMap = new HashMap<>();
         for (SchemaItem propertiesItem : schemaItem.getProperties()) {
             String propertiesItemKind = propertiesItem.getKind();
-            String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind,ElasticType.TEXT.getValue());
+            Object propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind,ElasticType.TEXT.getValue());
+            if(propertiesItem.getProperties() != null){
+                HashMap<String, Object> innerProperties = normalizeInnerProperties(propertiesItem);
+                propertiesElasticType = TypeMapper.getObjectsArrayMapping(propertiesItemKind, innerProperties);
+            }
             propertiesMap.put(propertiesItem.getPath(),propertiesElasticType);
         }
         return propertiesMap;

indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java

@@ -45,7 +45,7 @@ public class TypeMapper {
         metaAttributeIndexerType.put(RecordMetaAttribute.VERSION.getValue(), ElasticType.LONG.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.X_ACL.getValue(), ElasticType.KEYWORD.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.ACL.getValue(), getAclIndexerMapping());
-        metaAttributeIndexerType.put(RecordMetaAttribute.TAGS.getValue(), ElasticType.OBJECT.getValue());
+        metaAttributeIndexerType.put(RecordMetaAttribute.TAGS.getValue(), ElasticType.FLATTENED.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
         metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
         metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
@@ -104,7 +104,11 @@ public class TypeMapper {
             Map<String,Object> type = (Map<String, Object>) indexerType;
             Map<String, Object> propertiesMap = (Map<String, Object>) type.get(Constants.PROPERTIES);
             for (Map.Entry<String,Object> entry : propertiesMap.entrySet()){
-                entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build());
+                if(isMap(entry.getValue())){
+                    entry.setValue(getDataAttributeIndexerMapping(entry.getValue()));
+                }else {
+                    entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build());
+                }
             }
             return indexerType;
         }

provider/indexer-gcp/README.md

@@ -148,6 +148,8 @@ You will need to have the following environment variables defined.
 | `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
 | `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment |
 | `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
 
 
 **Entitlements configuration for integration accounts**

provider/indexer-gcp/pom.xml

@@ -25,7 +25,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-gcp</artifactId>
-            <version>0.7.0</version>
+            <version>0.9.0-SNAPSHOT</version>
         </dependency>
 
         <dependency>

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java

@@ -47,6 +47,7 @@ import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
 import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
 import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
 import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
 import org.springframework.web.context.annotation.RequestScope;
 
@@ -56,6 +57,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+@Primary
 @Component
 @RequestScope
 public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {

provider/indexer-gcp/src/main/resources/application.properties

@@ -42,3 +42,6 @@ security.https.certificate.trust=false
 indexer.que.service.mail=default@iam.gserviceaccount.com
 SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
 storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+
+google.audiences=123.apps.googleusercontent.com
+partition.api=http://localhost:8081/api/partition/v1
\ No newline at end of file

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

@@ -53,6 +53,11 @@ public class Steps extends SchemaServiceRecordSteps {
         super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
     }
 
+    @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+    public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+        super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+    }
+
     @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and  \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
     public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
         int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)