From 8ac99011f50e5b478250f09482f5670ae55e6125 Mon Sep 17 00:00:00 2001 From: Sviatoslav Nekhaienko Date: Mon, 18 Jan 2021 16:23:41 +0200 Subject: [PATCH 01/11] allOf has been improved --- .../schema/converter/PropertiesProcessor.java | 23 +++++++- .../converter/SchemaToStorageFormatImpl.java | 5 ++ .../schema/converter/tags/AllOfItem.java | 2 + .../schema/converter/tags/Definition.java | 2 + .../schema/converter/tags/PropertiesData.java | 2 + .../schema/converter/tags/TypeProperty.java | 2 + .../SchemaToStorageFormatImplTest.java | 31 ++++++++-- .../converter/root-properties/schema.json | 12 ++++ .../converter/root-properties/schema.json.res | 9 +++ .../tags/allOf/allOf-inside-allOf.json | 56 +++++++++++++++++++ .../tags/allOf/allOf-inside-allOf.json.res | 21 +++++++ .../tags/allOf/allOf-inside-property.json | 33 +++++++++++ .../tags/allOf/allOf-inside-property.json.res | 14 +++++ .../converter/tags/allOf/indefinitions.json | 29 ++++++++++ .../tags/allOf/indefinitions.json.res | 9 +++ pom.xml | 2 +- 16 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 indexer-core/src/test/resources/converter/root-properties/schema.json create mode 100644 indexer-core/src/test/resources/converter/root-properties/schema.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 6de10d52..3506c2ca 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -56,6 +56,10 @@ public class PropertiesProcessor { public Stream> processItem(AllOfItem allOfItem) { Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null"); + if (Objects.nonNull(allOfItem.getAllOf())) { + return allOfItem.getAllOf().stream().flatMap(this::processItem); + } + String ref = allOfItem.getRef(); return Objects.isNull(ref) ? @@ -85,7 +89,15 @@ public class PropertiesProcessor { new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef, "Unknown definition:" + definitionSubRef)); - return definition.getProperties().entrySet().stream().flatMap(this::processPropertyEntry); + if (Objects.nonNull(definition.getAllOf())) { + return definition.getAllOf().stream().flatMap(this::processItem); + } + + return processProperties(definition.getProperties()); + } + + public Stream> processProperties(Map properties){ + return properties.entrySet().stream().flatMap(this::processPropertyEntry); } private Stream> processPropertyEntry(Map.Entry entry) { @@ -106,6 +118,13 @@ public class PropertiesProcessor { return Stream.empty(); } + if (Objects.nonNull(entry.getValue().getAllOf())) { + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() + , log, new SchemaConverterPropertiesConfig()); + + return entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem); + } + if (Objects.nonNull(entry.getValue().getProperties())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() , log, new SchemaConverterPropertiesConfig()); @@ -163,7 +182,7 @@ public class PropertiesProcessor { private Supplier getFromFormat(Supplier formatSupplier){ return () -> { - String format = formatSupplier.get();; + String format = formatSupplier.get(); return Objects.nonNull(format) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(format, format) : null; }; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java index d35c8d25..1dc63193 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java @@ -101,6 +101,11 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat { storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef()) .collect(Collectors.toList())); } + + if (schemaData.getProperties() != null) { + storageSchemaItems.addAll(propertiesProcessor.processProperties(schemaData.getProperties()) + .collect(Collectors.toList())); + } } } else { log.warning("Schema doesn't have properties, kind:" + kind); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java index 9499e563..bef9a14b 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java @@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.List; import java.util.Map; @Data @@ -25,4 +26,5 @@ public class AllOfItem { private String ref; private String type; private Map properties; + private List allOf; } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java index f1b820f3..be8292a2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java @@ -16,9 +16,11 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import lombok.Data; +import java.util.List; import java.util.Map; @Data public class Definition { private Map properties; + private List allOf; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java index 706ee8b1..6c09c011 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java @@ -18,10 +18,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import java.util.List; +import java.util.Map; @Data public class PropertiesData { private List allOf; @JsonProperty("$ref") private String ref; + private Map properties; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index 0710bede..3053e5d2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.List; import java.util.Map; @Data @@ -28,4 +29,5 @@ public class TypeProperty { private String ref; private Items items; private Map properties; + private List allOf; } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java index f43d6261..c9009287 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java @@ -39,6 +39,8 @@ import static org.junit.Assert.fail; @SpringBootTest public class SchemaToStorageFormatImplTest { + private static final String KIND = "KIND_VAL"; + private ObjectMapper objectMapper = Jackson2ObjectMapperBuilder.json().build(); private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class); @@ -52,19 +54,24 @@ public class SchemaToStorageFormatImplTest { testSingleFile("/converter/basic/schema.json", "osdu:osdu:Wellbore:1.0.0"); } + @Test + public void rootProperties() { + testSingleFile("/converter/root-properties/schema.json", KIND); + } + @Test public void integrationTestSchema1() { - testSingleFile("/converter/integration-tests/index_records_1.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_1.schema", KIND); } @Test public void integrationTestSchema2() { - testSingleFile("/converter/integration-tests/index_records_2.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_2.schema", KIND); } @Test public void integrationTestSchema3() { - testSingleFile("/converter/integration-tests/index_records_3.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_3.schema", KIND); } @Test @@ -72,6 +79,21 @@ public class SchemaToStorageFormatImplTest { testSingleFile("/converter/wks/slb_wke_wellbore.json", "slb:wks:wellbore:1.0.6"); } + @Test + public void allOfInsideAllOf() { + testSingleFile("/converter/tags/allOf/allOf-inside-allOf.json", KIND); + } + + @Test + public void allOfInsideProperty() { + testSingleFile("/converter/tags/allOf/allOf-inside-property.json", KIND); + } + + @Test + public void allOfInDefinitions() { + testSingleFile("/converter/tags/allOf/indefinitions.json", KIND); + } + @Test public void folderPassed() throws URISyntaxException, IOException { @@ -85,8 +107,9 @@ public class SchemaToStorageFormatImplTest { private void testSingleFile(String filename, String kind) { String json = getSchemaFromSchemaService(filename); - Map expected = getStorageSchema( filename + ".res"); + Map converted = schemaToStorageFormatImpl.convertToMap(json, kind); + Map expected = getStorageSchema( filename + ".res"); compareSchemas(expected, converted, filename); } diff --git a/indexer-core/src/test/resources/converter/root-properties/schema.json b/indexer-core/src/test/resources/converter/root-properties/schema.json new file mode 100644 index 00000000..55fe1a30 --- /dev/null +++ b/indexer-core/src/test/resources/converter/root-properties/schema.json @@ -0,0 +1,12 @@ +{ + "properties": { + "data": { + "properties": { + "FacilityName": { + "type": "string" + } + }, + "type": "object" + } + } +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/root-properties/schema.json.res b/indexer-core/src/test/resources/converter/root-properties/schema.json.res new file mode 100644 index 00000000..cd12f24c --- /dev/null +++ b/indexer-core/src/test/resources/converter/root-properties/schema.json.res @@ -0,0 +1,9 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "FacilityName" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json new file mode 100644 index 00000000..8ba3af25 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json @@ -0,0 +1,56 @@ +{ + "definitions": { + "wellboreData1": { + "properties": { + "prop1": { + "type": "string" + } + } + }, + "wellboreData2": { + "properties": { + "prop2": { + "type": "string" + } + } + }, + "wellboreData3": { + "properties": { + "prop3": { + "type": "string" + } + } + }, + "wellboreData4": { + "properties": { + "prop4": { + "type": "string" + } + } + } + }, + "properties": { + "data": { + "allOf": [ + { + "allOf": [ + { + "$ref": "#/definitions/wellboreData1" + }, + { + "$ref": "#/definitions/wellboreData2" + } + ] + }, + { + "$ref": "#/definitions/wellboreData3" + }, + { + "$ref": "#/definitions/wellboreData4" + } + ] + } + } +} + + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res new file mode 100644 index 00000000..22a8e6a8 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res @@ -0,0 +1,21 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "prop1" + }, + { + "kind": "string", + "path": "prop2" + }, + { + "kind": "string", + "path": "prop3" + }, + { + "kind": "string", + "path": "prop4" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json new file mode 100644 index 00000000..1f2d761d --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json @@ -0,0 +1,33 @@ +{ + "definitions": { + "def1": { + "properties": { + "prop1": { + "type": "string" + } + } + } + }, + "properties": { + "data": { + "properties": { + "FacilityName": { + "allOf": [ + { + "$ref": "#/definitions/def1" + }, + { + "properties": { + "val" : { + "type": "string" } + } + } + ] + } + }, + "type": "object" + } + } +} + + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res new file mode 100644 index 00000000..147f31d3 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res @@ -0,0 +1,14 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "FacilityName.prop1" + }, + { + "kind": "string", + "path": "FacilityName.val" + } + + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json new file mode 100644 index 00000000..0064dd0a --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json @@ -0,0 +1,29 @@ +{ + "definitions": { + "wellboreData1": { + "properties": { + "prop1": { + "type": "string" + } + } + }, + "wellboreData2": { + "allOf": [ + { + "$ref": "#/definitions/wellboreData1" + } + ] + } + }, + "properties": { + "data": { + "type": "object", + "properties": { + "Field": { + "$ref": "#/definitions/wellboreData2" + } + } + } + } +} + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res new file mode 100644 index 00000000..e17ce90d --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res @@ -0,0 +1,9 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "Field.prop1" + } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8d6c8317..36b31e23 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ org.projectlombok lombok - 1.18.8 + 1.18.16 provided -- GitLab From ad88770d3c919320996f7ea7f6a61df4d80bf7d9 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Wed, 2 Dec 2020 17:42:56 +0400 Subject: [PATCH 02/11] Anthos --- pom.xml | 1 + provider/indexer-reference/Dockerfile | 5 + provider/indexer-reference/docker-compose.yml | 12 + .../deployment-os-indexer-service.yml | 179 ++++++ provider/indexer-reference/pom.xml | 197 +++++++ .../indexer/IndexerAnthosApplication.java | 40 ++ .../osdu/indexer/ServletInitializer.java | 29 + .../osdu/indexer/cache/AttributesCache.java | 63 +++ .../cache/DatastoreCredentialCache.java | 32 ++ .../cache/ElasticCredentialsCache.java | 62 +++ .../osdu/indexer/cache/IndexCache.java | 60 ++ .../osdu/indexer/cache/JwtCache.java | 62 +++ .../osdu/indexer/cache/KindsCache.java | 64 +++ .../osdu/indexer/cache/SchemaCache.java | 60 ++ .../indexer/di/EntitlementsClientFactory.java | 54 ++ .../osdu/indexer/di/RabbitMQFactoryImpl.java | 82 +++ .../osdu/indexer/di/TenantFactoryImpl.java | 97 ++++ .../opengroup/osdu/indexer/kms/KmsClient.java | 105 ++++ .../indexer/messagebus/IMessageFactory.java | 28 + .../osdu/indexer/middleware/IndexFilter.java | 121 ++++ .../persistence/DatastoreCredential.java | 119 ++++ .../indexer/persistence/DatastoreFactory.java | 70 +++ .../persistence/ElasticRepositoryMongoDB.java | 88 +++ .../indexer/persistence/MongoDdmsClient.java | 40 ++ .../osdu/indexer/publish/PublisherImpl.java | 103 ++++ .../security/GSuiteSecurityConfig.java | 46 ++ .../indexer/service/IndexerServiceImpl.java | 525 ++++++++++++++++++ .../indexer/util/DpsHeaderFactoryGcp.java | 60 ++ .../osdu/indexer/util/MongoClientHandler.java | 83 +++ .../osdu/indexer/util/RequestInfoImpl.java | 117 ++++ .../util/ServiceAccountJwtGcpClientImpl.java | 196 +++++++ .../main/resources/application-dev.properties | 40 ++ .../src/main/resources/application.properties | 78 +++ .../indexer/middleware/IndexFilterTest.java | 70 +++ .../indexer/service/CronServiceImplTest.java | 138 +++++ .../service/ElasticSettingServiceTest.java | 114 ++++ .../service/IndexCopyServiceImplTest.java | 193 +++++++ .../service/IndexerMappingServiceTest.java | 341 ++++++++++++ .../service/IndexerSchemaServiceTest.java | 403 ++++++++++++++ .../indexer/service/IndexerServiceTest.java | 354 ++++++++++++ .../indexer/service/ReindexServiceTest.java | 154 +++++ .../indexer/service/StorageServiceTest.java | 251 +++++++++ .../ServiceAccountJwtGcpClientImplTest.java | 171 ++++++ .../org.mockito.plugins.MockMaker | 1 + 44 files changed, 5108 insertions(+) create mode 100644 provider/indexer-reference/Dockerfile create mode 100644 provider/indexer-reference/docker-compose.yml create mode 100644 provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml create mode 100644 provider/indexer-reference/pom.xml create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java create mode 100644 provider/indexer-reference/src/main/resources/application-dev.properties create mode 100644 provider/indexer-reference/src/main/resources/application.properties create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java create mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java create mode 100644 provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/pom.xml b/pom.xml index 36b31e23..a85a2d46 100644 --- a/pom.xml +++ b/pom.xml @@ -136,6 +136,7 @@ provider/indexer-azure provider/indexer-gcp provider/indexer-ibm + provider/indexer-reference diff --git a/provider/indexer-reference/Dockerfile b/provider/indexer-reference/Dockerfile new file mode 100644 index 00000000..1e6901f3 --- /dev/null +++ b/provider/indexer-reference/Dockerfile @@ -0,0 +1,5 @@ +FROM openjdk:8-slim +WORKDIR /app +COPY target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar indexer-reference.jar +# Run the web service on container startup. +CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/indexer-reference.jar diff --git a/provider/indexer-reference/docker-compose.yml b/provider/indexer-reference/docker-compose.yml new file mode 100644 index 00000000..a6838447 --- /dev/null +++ b/provider/indexer-reference/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3" +services: + os-storage-app: + build: + args: + JAR_FILE: target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar + context: "" + dockerfile: ../Dockerfile + image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference + ports: + - "8080:8080" + diff --git a/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml new file mode 100644 index 00000000..0157d987 --- /dev/null +++ b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml @@ -0,0 +1,179 @@ +apiVersion: v1 +data: + AUTHORIZE_API: ${AUTHORIZE_API} + MONGO_DB_URL: ${MONGO_DB_URL} + MONGO_DB_USER: ${MONGO_DB_USER} + MONGO_DB_NAME: ${MONGO_DB_NAME} + REGION: ${REGION} + LEGALTAG_API: ${LEGALTAG_API} + INDEXER_HOST: ${INDEXER_HOST} + GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT} + STORAGE_HOSTNAME: ${STORAGE_HOSTNAME} + STORAGE_SCHEMA_HOST: ${STORAGE_SCHEMA_HOST} + STORAGE_QUERY_RECORD_HOST: ${STORAGE_QUERY_RECORD_HOST} + STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: ${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST} + INDEXER_QUEUE_HOST: ${INDEXER_QUEUE_HOST} + CRS_API: ${CRS_API} + REDIS_GROUP_HOST: ${REDIS_GROUP_HOST} + REDIS_SEARCH_HOST: ${REDIS_SEARCH_HOST} + GOOGLE_AUDIENCES: ${GOOGLE_AUDIENCES} + KEY_RING: ${KEY_RING} + KMS_KEY: ${KMS_KEY} +kind: ConfigMap +metadata: + labels: + app: indexer-reference + name: indexer-config + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + generateName: indexer-reference-anthos + labels: + app: indexer-reference + name: indexer-reference + namespace: default +spec: + selector: + matchLabels: + app: indexer-reference + replicas: 1 + template: + metadata: + labels: + app: indexer-reference + spec: + containers: + - env: + - name: AUTHORIZE_API + valueFrom: + configMapKeyRef: + key: AUTHORIZE_API + name: indexer-config + - name: MONGO_DB_URL + valueFrom: + configMapKeyRef: + key: MONGO_DB_URL + name: indexer-config + - name: MONGO_DB_USER + valueFrom: + configMapKeyRef: + key: MONGO_DB_USER + name: indexer-config + - name: MONGO_DB_PASSWORD + valueFrom: + secretKeyRef: + name: indexer-secret + key: mongo.db.password + - name: MONGO_DB_NAME + valueFrom: + configMapKeyRef: + key: MONGO_DB_NAME + name: indexer-config + - name: MB_RABBITMQ_URI + valueFrom: + secretKeyRef: + name: indexer-secret + key: mb.rabbitmq.uri + - name: REGION + valueFrom: + configMapKeyRef: + key: REGION + name: indexer-config + - name: LEGALTAG_API + valueFrom: + configMapKeyRef: + key: LEGALTAG_API + name: indexer-config + - name: INDEXER_HOST + valueFrom: + configMapKeyRef: + key: INDEXER_HOST + name: indexer-config + - name: GOOGLE_CLOUD_PROJECT + valueFrom: + configMapKeyRef: + key: GOOGLE_CLOUD_PROJECT + name: indexer-config + - name: STORAGE_HOSTNAME + valueFrom: + configMapKeyRef: + key: STORAGE_HOSTNAME + name: indexer-config + - name: STORAGE_SCHEMA_HOST + valueFrom: + configMapKeyRef: + key: STORAGE_SCHEMA_HOST + name: indexer-config + - name: STORAGE_QUERY_RECORD_HOST + valueFrom: + configMapKeyRef: + key: STORAGE_QUERY_RECORD_HOST + name: indexer-config + - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST + valueFrom: + configMapKeyRef: + key: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST + name: indexer-config + - name: INDEXER_QUEUE_HOST + valueFrom: + configMapKeyRef: + key: INDEXER_QUEUE_HOST + name: indexer-config + - name: CRS_API + valueFrom: + configMapKeyRef: + key: CRS_API + name: indexer-config + - name: REDIS_GROUP_HOST + valueFrom: + configMapKeyRef: + key: REDIS_GROUP_HOST + name: indexer-config + - name: REDIS_SEARCH_HOST + valueFrom: + configMapKeyRef: + key: REDIS_SEARCH_HOST + name: indexer-config + - name: GOOGLE_AUDIENCES + valueFrom: + configMapKeyRef: + key: GOOGLE_AUDIENCES + name: indexer-config + - name: KEY_RING + valueFrom: + configMapKeyRef: + key: KEY_RING + name: indexer-config + - name: KMS_KEY + valueFrom: + configMapKeyRef: + key: KMS_KEY + name: indexer-config + image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference:9966597-dirty + name: indexer-reference +--- +apiVersion: v1 +kind: Service +metadata: + name: indexer-reference + namespace: default +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + selector: + app: indexer-reference + type: LoadBalancer +--- +apiVersion: v1 +data: + mongo.db.password: ${mongo.db.password} + mb.rabbitmq.uri: ${mb.rabbitmq.uri} +kind: Secret +metadata: + name: indexer-secret + namespace: default +type: Opaque \ No newline at end of file diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml new file mode 100644 index 00000000..99d01082 --- /dev/null +++ b/provider/indexer-reference/pom.xml @@ -0,0 +1,197 @@ + + + + + 4.0.0 + + + org.opengroup.osdu.indexer + indexer-service + 1.0.4-SNAPSHOT + ../../pom.xml + + + indexer-reference + 0.0.1-SNAPSHOT + indexer-reference + Indexer Service GCP Anthos + jar + + + + org.opengroup.osdu.indexer + indexer-core + 1.0.6-SNAPSHOT + + + + org.opengroup.osdu + core-lib-gcp + 0.1.17 + + + + org.opengroup.osdu + os-core-common + 0.0.18 + + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + + com.google.cloud + google-cloud-logging + 1.72.0 + + + com.google.apis + google-api-services-storage + v1-rev150-1.25.0 + + + com.google.guava + guava-jdk5 + + + + + com.google.apis + google-api-services-cloudkms + v1-rev81-1.25.0 + + + + org.springframework.boot + spring-boot-starter-amqp + + + org.elasticsearch + elasticsearch + 6.6.2 + + + org.elasticsearch.client + elasticsearch-rest-client + 6.6.2 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.2 + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.security + spring-security-test + test + + + + + junit + junit + 4.12 + test + + + + org.powermock + powermock-api-mockito2 + 2.0.2 + test + + + + org.powermock + powermock-module-junit4 + 2.0.2 + test + + + org.mockito + mockito-core + 2.26.0 + test + + + org.codehaus.mojo + cobertura-maven-plugin + 2.7 + test + + + + com.auth0 + java-jwt + 3.8.1 + + + + + com.github.stefanbirkner + system-rules + 1.2.0 + test + + + com.google.api.grpc + proto-google-iam-v1 + 0.12.0 + + + org.springframework.boot + spring-boot-actuator-autoconfigure + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + spring-boot + + org.opengroup.osdu.indexer.IndexerAnthosApplication + + + + + + + + + + diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java new file mode 100644 index 00000000..a87c9d45 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer; + +import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; + +@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) +@Configuration +@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) +public class IndexerAnthosApplication { + + public static void main(String[] args) { + SpringApplication.run(IndexerAnthosApplication.class, args); + } + +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java new file mode 100644 index 00000000..ccc93114 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer; + +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; + +public class ServletInitializer extends SpringBootServletInitializer { + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(IndexerAnthosApplication.class); + } +} + diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java new file mode 100644 index 00000000..bdfa51cd --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import java.util.Set; + +@Component +public class AttributesCache implements IAttributesCache, AutoCloseable { + + private RedisCache cache; + + public AttributesCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { + + cache = new RedisCache(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); + } + + @Override + public void put(String key, Set value) { + this.cache.put(key, value); + } + + @Override + public Set get(String key) { + return this.cache.get(key); + } + + @Override + public void delete(String key) { + this.cache.delete(key); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } + + @Override + public void close() { + this.cache.close(); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java new file mode 100644 index 00000000..d17e26f7 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import com.google.auth.oauth2.AccessToken; +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class DatastoreCredentialCache extends RedisCache { + + // Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration + public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { + super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java new file mode 100644 index 00000000..7a5d5f6f --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class ElasticCredentialsCache implements IElasticCredentialsCache, AutoCloseable { + + private RedisCache cache; + + public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, String.class, ClusterSettings.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, ClusterSettings o) { + this.cache.put(s,o); + } + + @Override + public ClusterSettings get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java new file mode 100644 index 00000000..d9b7197f --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class IndexCache implements IIndexCache, AutoCloseable { + private RedisCache cache; + + public IndexCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Boolean o) { + this.cache.put(s, o); + } + + @Override + public Boolean get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java new file mode 100644 index 00000000..2b442b62 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.model.search.IdToken; +import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class JwtCache implements IJwtCache, AutoCloseable { + RedisCache cache; + + // google service account id_token can be requested only for 1 hr + private final static int EXPIRED_AFTER = 59; + + public JwtCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + EXPIRED_AFTER * 60, String.class, IdToken.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, IdToken o) { + this.cache.put(s, o); + } + + @Override + public IdToken get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java new file mode 100644 index 00000000..d70210a0 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Set; + +@Component +public class KindsCache implements IKindsCache, AutoCloseable { + private RedisCache cache; + + public KindsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION, + @Value("${KINDS_REDIS_DATABASE}") final String KINDS_REDIS_DATABASE) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60, + Integer.parseInt(KINDS_REDIS_DATABASE), String.class, Set.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Set o) { + this.cache.put(s, o); + } + + @Override + public Set get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java new file mode 100644 index 00000000..1fc8b411 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class SchemaCache implements ISchemaCache, AutoCloseable { + private RedisCache cache; + + public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, + @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, + @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) { + cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), + Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, String o) { + this.cache.put(s, o); + } + + @Override + public String get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java new file mode 100644 index 00000000..8460aa5f --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig; +import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory; +import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +@Lazy +public class EntitlementsClientFactory extends AbstractFactoryBean { + + @Value("${AUTHORIZE_API}") + private String AUTHORIZE_API; + + @Value("${AUTHORIZE_API_KEY:}") + private String AUTHORIZE_API_KEY; + + @Override + protected IEntitlementsFactory createInstance() throws Exception { + + return new EntitlementsFactory(EntitlementsAPIConfig + .builder() + .rootUrl(AUTHORIZE_API) + .apiKey(AUTHORIZE_API_KEY) + .build()); + } + + @Override + public Class getObjectType() { + return IEntitlementsFactory.class; + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java new file mode 100644 index 00000000..20236aaf --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; +import javax.annotation.PostConstruct; +import org.opengroup.osdu.indexer.messagebus.IMessageFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RabbitMQFactoryImpl implements IMessageFactory { + + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + + @Value("${mb.rabbitmq.uri}") + private String uri; + + private Channel channel; + + @PostConstruct + private void init() { + ConnectionFactory factory = new ConnectionFactory(); + try { + LOG.debug("RabbitMQ Channel " + uri); + factory.setUri(uri); + factory.setAutomaticRecoveryEnabled(true); + Connection conn = factory.newConnection(); + this.channel = conn.createChannel(); + LOG.debug("RabbitMQ Channel was created."); + for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { + channel.queueDeclare(queue, true, false, false, null); + LOG.debug("Queue [" + queue + "] was declared."); + } + } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { + LOG.error(e.getMessage(), e); + } + + } + + @Override + public void sendMessage(String msg) { + this.sendMessage("records", msg); + } + + @Override + public void sendMessage(String queueName, String msg) { + String queueNameWithPrefix = queueName; + try { + channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); + LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); + } catch (IOException e) { + LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]"); + LOG.error(e.getMessage(), e); + } + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java new file mode 100644 index 00000000..8aad1a1a --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.di; + + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.bson.Document; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; +import org.opengroup.osdu.indexer.persistence.MongoDdmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +@Primary +@Component +public class TenantFactoryImpl implements ITenantFactory { + + private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); + + public static final String MAIN_DATABASE = "main"; + public static final String TENANT_INFO = "tenantinfo"; + + @Autowired + private MongoDdmsClient mongoClient; + + private Map tenants; + + public boolean exists(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.containsKey(tenantName); + } + + public TenantInfo getTenantInfo(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.get(tenantName); + } + + public Collection listTenantInfo() { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.values(); + } + + public ICache createCache(String tenantName, String host, int port, + int expireTimeSeconds, Class classOfV) { + return null; + } + + public void flushCache() { + } + + private void initTenants() { + this.tenants = new HashMap<>(); + MongoCollection mongoCollection = mongoClient + .getMongoCollection(MAIN_DATABASE, TENANT_INFO); + FindIterable results = mongoCollection.find(); + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + } + for (Document document : results) { + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class); + this.tenants.put(tenantInfo.getName(), tenantInfo); + } + } + +} + diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java new file mode 100644 index 00000000..2adc6336 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.kms; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.cloudkms.v1.CloudKMS; +import com.google.api.services.cloudkms.v1.CloudKMSScopes; +import com.google.api.services.cloudkms.v1.model.DecryptRequest; +import com.google.api.services.cloudkms.v1.model.DecryptResponse; +import com.google.api.services.cloudkms.v1.model.EncryptRequest; +import com.google.api.services.cloudkms.v1.model.EncryptResponse; +import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; +import org.opengroup.osdu.core.common.search.Preconditions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +@Component +@RequestScope +public class KmsClient implements IKmsClient { + + @Value("${GOOGLE_CLOUD_PROJECT}") + private String GOOGLE_CLOUD_PROJECT; + + @Value("${KMS_KEY}") + private String KMS_KEY; + + @Value("${KEY_RING}") + private String KEY_RING; + + private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s"; + + /** + * Encrypts the given plaintext using the specified crypto key. + * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation + */ + public String encryptString(String textToBeEncrypted) throws IOException { + Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null"); + + byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8); + String resourceName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); + CloudKMS kms = createAuthorizedClient(); + EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext); + EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys() + .encrypt(resourceName, request) + .execute(); + return response.getCiphertext(); + } + + /** + * Decrypts the provided ciphertext with the specified crypto key. + * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled + */ + public String decryptString(String textToBeDecrypted) throws IOException { + Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null"); + + CloudKMS kms = createAuthorizedClient(); + String cryptoKeyName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); + DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted); + DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys() + .decrypt(cryptoKeyName, request) + .execute(); + return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim(); + } + + /** + * Creates an authorized CloudKMS client service using Application Default Credentials. + * + * @return an authorized CloudKMS client + * @throws IOException if there's an error getting the default credentials. + */ + private CloudKMS createAuthorizedClient() throws IOException { + HttpTransport transport = new NetHttpTransport(); + JsonFactory jsonFactory = new JacksonFactory(); + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + credential = credential.createScoped(CloudKMSScopes.all()); + } + return new CloudKMS.Builder(transport, jsonFactory, credential) + .setApplicationName("CloudKMS snippets") + .build(); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java new file mode 100644 index 00000000..1679a6d6 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.messagebus; + +public interface IMessageFactory { + String DEFAULT_QUEUE_NAME = "records"; + String LEGAL_QUEUE_NAME = "legal"; + String INDEXER_QUEUE_NAME = "indexer"; + + void sendMessage(String msg); + + void sendMessage(String queueName, String msg); +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java new file mode 100644 index 00000000..95bbd9be --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.middleware; + +import com.google.common.base.Strings; +import lombok.extern.java.Log; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.http.ResponseHeaders; +import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +@Log +@Component +public class IndexFilter implements Filter { + + @Inject + private DpsHeaders dpsHeaders; + + @Inject + private IRequestInfo requestInfo; + + @Value("${DEPLOYMENT_ENVIRONMENT}") + private String DEPLOYMENT_ENVIRONMENT; + + private FilterConfig filterConfig; + + private static final String PATH_SWAGGER = "/swagger.json"; + private static final String PATH_TASK_HANDLERS = "task-handlers"; + private static final String PATH_CRON_HANDLERS = "cron-handlers"; + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + String uri = httpRequest.getRequestURI().toLowerCase(); + + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) { + if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) != DeploymentEnvironment.LOCAL) { + checkWorkerApiAccess(requestInfo); + } + } + + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) { + checkWorkerApiAccess(requestInfo); + } + +// if (!httpRequest.isSecure()) { +// throw new AppException(302, "Redirect", "HTTP is not supported. Use HTTPS."); +// } + + filterChain.doFilter(servletRequest, servletResponse); + + HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; + Map> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; + for (Map.Entry> header : standardHeaders.entrySet()) { + httpResponse.addHeader(header.getKey(), header.getValue().toString()); + } + if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) { + httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId()); + } + + } + + @Override + public void destroy() { + } + + private void checkWorkerApiAccess(IRequestInfo requestInfo) { +// if (requestInfo.isTaskQueueRequest()) return; +// throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only"); + } + + private List validateAccountId(DpsHeaders requestHeaders) { + String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId(); + String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader); + + if (Strings.isNullOrEmpty(accountHeader)) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); + } + + List dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*")); + if (dataPartitions.isEmpty() || dataPartitions.size() > 1) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); + } + return dataPartitions; + } + +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java new file mode 100644 index 00000000..c3285bd7 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.persistence; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.iam.v1.Iam; +import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt; +import com.google.api.services.iam.v1.IamScopes; +import com.google.api.services.iam.v1.model.SignJwtRequest; +import com.google.api.services.iam.v1.model.SignJwtResponse; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.gson.JsonObject; +import java.util.Collections; +import java.util.Date; +import org.apache.commons.lang3.time.DateUtils; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.util.Crc32c; +import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; + +public class DatastoreCredential extends GoogleCredentials { + + private static final long serialVersionUID = 8344377091688956815L; + private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + private Iam iam; + + private final TenantInfo tenant; + private final DatastoreCredentialCache cache; + + protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { + this.tenant = tenant; + this.cache = cache; + } + + @Override + public AccessToken refreshAccessToken() { + + String cacheKey = this.getCacheKey(); + + AccessToken accessToken = this.cache.get(cacheKey); + + if (accessToken != null) { + return accessToken; + } + + try { + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(this.getPayload()); + + String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); + + SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest); + + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); + + accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); + + this.cache.put(cacheKey, accessToken); + + return accessToken; + } catch (Exception e) { + throw new RuntimeException("Error creating datastore credential", e); + } + } + + private String getPayload() { + JsonObject payload = new JsonObject(); + payload.addProperty("iss", this.tenant.getServiceAccount()); + payload.addProperty("sub", this.tenant.getServiceAccount()); + payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); + payload.addProperty("iat", System.currentTimeMillis() / 1000); + + return payload.toString(); + } + + protected void setIam(Iam iam) { + this.iam = iam; + } + + private Iam getIam() throws Exception { + if (this.iam == null) { + + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); + } + + Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY, + credential).setApplicationName("Search Service"); + + this.iam = builder.build(); + } + return this.iam; + } + + + private String getCacheKey() { + return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java new file mode 100644 index 00000000..db90ee7e --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.persistence; + +import com.google.api.gax.retrying.RetrySettings; +import com.google.cloud.TransportOptions; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.http.HttpTransportOptions; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; +import org.springframework.stereotype.Component; +import org.threeten.bp.Duration; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +@Component +public class DatastoreFactory { + + @Inject + private DatastoreCredentialCache cache; + + private static Map DATASTORE_CLIENTS = new HashMap<>(); + + private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() + .setMaxAttempts(6) + .setInitialRetryDelay(Duration.ofSeconds(10)) + .setMaxRetryDelay(Duration.ofSeconds(32)) + .setRetryDelayMultiplier(2.0) + .setTotalTimeout(Duration.ofSeconds(50)) + .setInitialRpcTimeout(Duration.ofSeconds(50)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeout(Duration.ofSeconds(50)) + .build(); + + private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() + .setReadTimeout(30000) + .build(); + + public Datastore getDatastoreInstance(TenantInfo tenantInfo) { + if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { + Datastore googleDatastore = DatastoreOptions.newBuilder() + .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) + .setRetrySettings(RETRY_SETTINGS) + .setTransportOptions(TRANSPORT_OPTIONS) + .setNamespace(tenantInfo.getName()) + .setProjectId(tenantInfo.getProjectId()) + .build().getService(); + DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); + } + return DATASTORE_CLIENTS.get(tenantInfo.getName()); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java new file mode 100644 index 00000000..a3ab89c5 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.persistence; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.Objects; +import org.apache.http.HttpStatus; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; +import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ElasticRepositoryMongoDB implements IElasticRepository { + + private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class); + + private static final String SCHEMA_DATABASE = "local"; + private static final String SEARCH_SETTINGS = "SearchSettings"; + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; + + @Autowired + private MongoDdmsClient mongoClient; + + @Autowired + private IKmsClient kmsClient; + + @Override + public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { + MongoCollection mongoCollection = mongoClient + .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS); + + FindIterable results = mongoCollection.find(); + + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database."); + } + + Document document = results.first(); + + String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString(); + String encryptedHost = document.get(HOST).toString(); + String encryptedPort = document.get(PORT).toString(); + + try { + + String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost); + String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort); + String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration); + + int port = Integer.parseInt(portString); + ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword); + clusterSettings.setHttps(false); + return clusterSettings; + + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database.", e); + } + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java new file mode 100644 index 00000000..43b3fd5b --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.persistence; + +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.indexer.util.MongoClientHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MongoDdmsClient { + + @Autowired + private MongoClientHandler mongoClientHandler; + + @Autowired + private TenantInfo tenantInfo; + + public MongoCollection getMongoCollection(String dbName, String collectionName) { + return mongoClientHandler.getMongoClient().getDatabase(dbName) + .getCollection(collectionName); + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java new file mode 100644 index 00000000..d2a4d74c --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java @@ -0,0 +1,103 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.publish; + + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.common.Strings; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.core.common.model.indexer.RecordStatus; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.indexer.messagebus.IMessageFactory; +import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class PublisherImpl implements IPublisher { + + private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class); + + @Autowired + IMessageFactory mq; + + + @Override + public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { + + String tenant = headers.getPartitionId(); + if (Strings.isNullOrEmpty(tenant)) { + tenant = headers.getAccountId(); + } + + Map message = new HashMap<>(); + message.put(tenant, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus); + message.put("data", recordChangedMessages.toString()); + + try { + LOG.info("Indexer publishes message " + headers.getCorrelationId()); + mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) { + + Gson gson = new GsonBuilder().create(); + Map attributesMap = new HashMap<>(); + Type listType = new TypeToken>() { + }.getType(); + + JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType); + String statusChangedTagsData = (statusChangedTagsJson.toString()); + + String tenant = headers.getPartitionId(); + // This code it to provide backward compatibility to slb-account-id + if (!Strings.isNullOrEmpty(tenant)) { + attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + } else { + attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + } + headers.addCorrelationIdIfMissing(); + attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = new RecordChangedMessages(); + // statusChangedTagsData is not ByteString but String + recordChangedMessages.setData(statusChangedTagsData); + recordChangedMessages.setAttributes(attributesMap); + + return recordChangedMessages; + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java new file mode 100644 index 00000000..7927d6f3 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.security; + +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.builders.WebSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +@Configuration +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity http) throws Exception { + + http.httpBasic().disable() + .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy + + } + + @Override + public void configure(WebSecurity web) throws Exception { + web.ignoring().antMatchers("/api-docs") + .antMatchers("/swagger"); + } + +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java new file mode 100644 index 00000000..0b558b08 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -0,0 +1,525 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.service; + +import com.google.gson.Gson; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import javax.inject.Inject; +import org.apache.http.HttpStatus; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.opengroup.osdu.core.common.Constants; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.entitlements.Acl; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.RequestStatus; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.model.indexer.RecordIndexerPayload; +import org.opengroup.osdu.core.common.model.indexer.RecordInfo; +import org.opengroup.osdu.core.common.model.indexer.RecordStatus; +import org.opengroup.osdu.core.common.model.indexer.Records; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; +import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; + +@Service +@Primary +public class IndexerServiceImpl implements IndexerService { + + private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); + + private static final List RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE)); + + private final Gson gson = new Gson(); + + @Inject + private JaxRsDpsLog jaxRsDpsLog; + @Inject + private AuditLogger auditLogger; + @Inject + private StorageService storageService; + @Inject + private IndexSchemaService schemaService; + @Inject + private IndicesService indicesService; + @Inject + private IndexerMappingService mappingService; + @Inject + private IPublisher progressPublisher; + @Inject + private ElasticClientHandler elasticClientHandler; + @Inject + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Inject + private ElasticIndexNameResolver elasticIndexNameResolver; + @Inject + private StorageIndexerPayloadMapper storageIndexerPayloadMapper; + @Inject + private IRequestInfo requestInfo; + @Inject + private JobStatus jobStatus; + + private DpsHeaders headers; + + @Override + public JobStatus processRecordChangedMessages(RecordChangedMessages message, List recordInfos) throws Exception { + + // this should not happen + if (recordInfos.size() == 0) return null; + + String errorMessage = ""; + List retryRecordIds = new LinkedList<>(); + + // get auth header with service account Authorization + this.headers = this.requestInfo.getHeaders(); + String authorization = message.getAttributes().get("authorization"); + headers.put("authorization", authorization); + // initialize status for all messages. + this.jobStatus.initialize(recordInfos); + + try { + // get upsert records + Map> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos); + if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) { + List upsertFailureRecordIds = processUpsertRecords(upsertRecordMap); + retryRecordIds.addAll(upsertFailureRecordIds); + } + + // get delete records + Map> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos); + if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) { + List deleteFailureRecordIds = processDeleteRecords(deleteRecordMap); + retryRecordIds.addAll(deleteFailureRecordIds); + } + + // process schema change messages + Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + this.schemaService.processSchemaMessages(schemaMsgs); + } + + // process failed records + if (retryRecordIds.size() > 0) { + retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message); + } + } catch (IOException e) { + errorMessage = e.getMessage(); + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e); + } catch (AppException e) { + errorMessage = e.getMessage(); + throw e; + } catch (Exception e) { + errorMessage = "error indexing records"; + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e); + } finally { + this.jobStatus.finalizeRecordStatus(errorMessage); + this.updateAuditLog(); + this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus); + } + + return jobStatus; + } + + @Override + public void processSchemaMessages(List recordInfos) throws IOException { + Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { + schemaMsgs.entrySet().forEach(msg -> { + try { + processSchemaEvents(restClient, msg); + } catch (IOException | ElasticsearchStatusException e) { + throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage()); + } + }); + } + } + } + + private void processSchemaEvents(RestHighLevelClient restClient, + Map.Entry msg) throws IOException, ElasticsearchStatusException { + String kind = msg.getKey(); + String index = elasticIndexNameResolver.getIndexNameFromKind(kind); + + boolean indexExist = indicesService.isIndexExist(restClient, index); + if (indexExist && msg.getValue() == OperationType.purge_schema) { + indicesService.deleteIndex(restClient, index); + } + } + + private List processUpsertRecords(Map> upsertRecordMap) throws Exception { + // get schema for kind + Map schemas = this.getSchema(upsertRecordMap); + + if (schemas.isEmpty()) return new LinkedList<>(); + + // get recordIds with valid upsert index-status + List recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus(); + + if (recordIds.isEmpty()) return new LinkedList<>(); + + // get records via storage api + Records storageRecords = this.storageService.getStorageRecords(recordIds); + List failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords()); + + // map storage records to indexer payload + RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords); + + jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size())); + + // index records + failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload)); + + return failedOrRetryRecordIds; + } + + private Map getSchema(Map> upsertRecordMap) { + + Map schemas = new HashMap<>(); + + try { + for (Map.Entry> entry : upsertRecordMap.entrySet()) { + + String kind = entry.getKey(); + IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); + if (schemaObj.isDataSchemaMissing()) { + this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind)); + } + + schemas.put(kind, schemaObj); + } + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e); + } + + return schemas; + } + + private RecordIndexerPayload getIndexerPayload(Map> upsertRecordMap, Map kindSchemaMap, Records records) { + List storageValidRecords = records.getRecords(); + List indexerPayload = new ArrayList<>(); + List schemas = new ArrayList<>(); + + for (Records.Entity storageRecord : storageValidRecords) { + + Map idOperationMap = upsertRecordMap.get(storageRecord.getKind()); + + // skip if storage returned record with same id but different kind + if (idOperationMap == null) { + String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind()); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId())); + continue; + } + + IndexSchema schema = kindSchemaMap.get(storageRecord.getKind()); + schemas.add(schema); + + // skip indexing of records if data block is empty + RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap); + if (document != null) { + indexerPayload.add(document); + } + } + + // this should only happen if storage service returned WRONG records with kind for all the records in the messages + if (indexerPayload.isEmpty()) { + throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records"); + } + + return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build(); + } + + private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map idToOperationMap) { + + RecordIndexerPayload.Record document = null; + + try { + Map storageRecordData = storageRecord.getData(); + document = new RecordIndexerPayload.Record(); + if (storageRecordData == null || storageRecordData.isEmpty()) { + String message = "empty or null data block found in the storage record"; + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); + } else if (schemaObj.isDataSchemaMissing()) { + document.setSchemaMissing(true); + } else { + Map dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId()); + if (dataMap.isEmpty()) { + document.setMappingMismatch(true); + String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); + } + document.setData(dataMap); + } + } catch (AppException e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e); + } + + try { + // index individual parts of kind + String[] kindParts = storageRecord.getKind().split(":"); + + document.setKind(storageRecord.getKind()); + document.setNamespace(kindParts[0] + ":" + kindParts[1]); + document.setType(kindParts[2]); + document.setId(storageRecord.getId()); + document.setVersion(storageRecord.getVersion()); + document.setAcl(storageRecord.getAcl()); + document.setLegal(storageRecord.getLegal()); + RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); + if (recordStatus.getIndexProgress().getStatusCode() == 0) { + recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); + } + document.setIndexProgress(recordStatus.getIndexProgress()); + if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry()); + document.setOperationType(idToOperationMap.get(storageRecord.getId())); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e); + } + return document; + } + + private List processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception { + + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + List schemas = recordIndexerPayload.getSchemas(); + if (schemas == null || schemas.isEmpty()) { + return new LinkedList<>(); + } + + // process the schema + this.cacheOrCreateElasticMapping(schemas, restClient); + + // process the records + return this.upsertRecords(recordIndexerPayload.getRecords(), restClient); + } + } + + private void cacheOrCreateElasticMapping(List schemas, RestHighLevelClient restClient) throws Exception { + + for (IndexSchema schema : schemas) { + String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind()); + + // check if index exist + if (this.indicesService.isIndexExist(restClient, index)) continue; + + // create index + Map mapping = this.mappingService.getIndexMappingFromRecordSchema(schema); + if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index)); + } + } + } + + private List upsertRecords(List records, RestHighLevelClient restClient) throws AppException { + if (records == null || records.isEmpty()) return new LinkedList<>(); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + + for (RecordIndexerPayload.Record record : records) { + if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) { + // it will come here when schema is missing + // TODO: rollback once we know what is causing the problem + jaxRsDpsLog.warning(String.format("data not found for record: %s", record)); + } + + OperationType operation = record.getOperationType(); + Map sourceMap = getSourceMap(record); + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); + + if (operation == OperationType.create) { + IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(indexRequest); + } else if (operation == OperationType.update) { + UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(updateRequest); + } + } + + return processBulkRequest(restClient, bulkRequest); + } + + private List processDeleteRecords(Map> deleteRecordMap) throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + + for (Map.Entry> record : deleteRecordMap.entrySet()) { + + String[] kindParts = record.getKey().split(":"); + String type = kindParts[2]; + + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); + + for (String id : record.getValue()) { + DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + bulkRequest.add(deleteRequest); + } + } + + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + return processBulkRequest(restClient, bulkRequest); + } + } + + private List processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException { + + List failureRecordIds = new LinkedList<>(); + if (bulkRequest.numberOfActions() == 0) return failureRecordIds; + + + + try { + BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); + + // log failed bulk requests + ArrayList bulkFailures = new ArrayList<>(); + int succeededResponses = 0; + int failedResponses = 0; + + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage())); + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage()); + if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) { + failureRecordIds.add(bulkItemResponse.getId()); + } + failedResponses++; + } else { + succeededResponses++; + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully"); + } + } + if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures); + + jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses)); + } catch (IOException e) { + // throw explicit 504 for IOException + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e); + } catch (ElasticsearchStatusException e) { + throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e); + } + return failureRecordIds; + } + + private Map getSourceMap(RecordIndexerPayload.Record record) { + + Map indexerPayload = new HashMap<>(); + + // get the key and get the corresponding object from the individualRecord object + if (record.getData() != null) { + Map data = new HashMap<>(); + for (Map.Entry entry : record.getData().entrySet()) { + data.put(entry.getKey(), entry.getValue()); + } + indexerPayload.put(Constants.DATA, data); + } + + indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId()); + indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind()); + indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace()); + indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); + indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); + indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); + indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); + indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); + indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); + if (record.getAncestry() != null) { + indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry()); + } + return indexerPayload; + } + + private void retryAndEnqueueFailedRecords(List recordInfos, List failuresRecordIds, RecordChangedMessages message) throws IOException { + + jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds)); + List retryRecordInfos = new LinkedList<>(); + for (String recordId : failuresRecordIds) { + for (RecordInfo origMessage : recordInfos) { + if (origMessage.getId().equalsIgnoreCase(recordId)) { + retryRecordInfos.add(origMessage); + } + } + } + + RecordChangedMessages newMessage = RecordChangedMessages.builder() + .messageId(message.getMessageId()) + .publishTime(message.getPublishTime()) + .data(this.gson.toJson(retryRecordInfos)) + .attributes(message.getAttributes()).build(); + + String payLoad = this.gson.toJson(newMessage); + this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers); + } + + private void updateAuditLog() { + logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail); + logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail); + logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail); + logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail); + } + + private void logAuditEvents(OperationType operationType, Consumer> successEvent, Consumer> failedEvent) { + List succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType); + if(!succeededRecords.isEmpty()) { + successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList())); + } + List skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType); + List failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType); + failedRecords.addAll(skippedRecords); + if(!failedRecords.isEmpty()) { + failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList())); + } + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java new file mode 100644 index 00000000..0e3c2016 --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.util; + +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.inject.Inject; + +import com.google.common.base.Strings; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; + +import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; +import org.opengroup.osdu.core.gcp.util.TraceIdExtractor; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +@Primary +public class DpsHeaderFactoryGcp extends DpsHeaders { + + @Inject + public DpsHeaderFactoryGcp(HttpServletRequest request) { + + Map headers = Collections + .list(request.getHeaderNames()) + .stream() + .collect(Collectors.toMap(h -> h, request::getHeader)); + + String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); + + if(!Strings.isNullOrEmpty(traceContext)){ + headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); + } + + this.addFromMap(headers); + + // Add Correlation ID if missing + this.addCorrelationIdIfMissing(); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java new file mode 100644 index 00000000..b60ef1da --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.util; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class MongoClientHandler { + + private static final Logger LOG = LoggerFactory.getLogger( + org.opengroup.osdu.indexer.util.MongoClientHandler.class); + private static final String MONGO_PREFIX = "mongodb://"; + private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; + + private MongoClient mongoClient = null; + + @Value("${mongo.db.url:#{null}}") + private String dbUrl; + + @Value("${mongo.db.user:#{null}}") + private String dbUser; + + @Value("${mongo.db.password:#{null}}") + private String dbPassword; + + private MongoClient getOrInitMongoClient() throws RuntimeException { + if (mongoClient != null) { + return mongoClient; + } + + final String connectionString = String.format("%s%s:%s@%s/?%s", + MONGO_PREFIX, + dbUser, + dbPassword, + dbUrl, + MONGO_OPTIONS); + ConnectionString connString = new ConnectionString(connectionString); + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connString) + .retryWrites(true) + .build(); + try { + mongoClient = MongoClients.create(settings); + } catch (Exception ex) { + LOG.error("Error connecting MongoDB", ex); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB", + ex.getMessage(), ex); + } + return mongoClient; + } + + public MongoClient getMongoClient() { + if (mongoClient == null) { + getOrInitMongoClient(); + } + return mongoClient; + } + +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java new file mode 100644 index 00000000..2396dd1e --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.util; + +import com.google.common.base.Strings; +import lombok.extern.java.Log; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.Constants; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; +import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +import javax.inject.Inject; +import java.util.Map; + +import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; + + +@Log +@Component +@RequestScope +public class RequestInfoImpl implements IRequestInfo { + + @Inject + private DpsHeaders dpsHeaders; + + @Inject + private IServiceAccountJwtClient serviceAccountJwtClient; + + @Inject + private TenantInfo tenantInfo; + + @Value("${DEPLOYMENT_ENVIRONMENT}") + private String DEPLOYMENT_ENVIRONMENT; + + private static final String expectedCronHeaderValue = "true"; + + @Override + public DpsHeaders getHeaders() { + + return this.dpsHeaders; + } + + @Override + public String getPartitionId() { + return this.dpsHeaders.getPartitionId(); + } + + @Override + public Map getHeadersMap() { + return this.dpsHeaders.getHeaders(); + } + + @Override + public Map getHeadersMapWithDwdAuthZ() { + return getHeadersWithDwdAuthZ().getHeaders(); + } + + @Override + public DpsHeaders getHeadersWithDwdAuthZ() { + // Update DpsHeaders so that service account creds are passed down + this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader()); + return this.dpsHeaders; + } + + @Override + public boolean isCronRequest() { + String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null); + return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); + } + + @Override + public boolean isTaskQueueRequest() { + if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false; + + String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); + return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); + } + + public String checkOrGetAuthorizationHeader() { + if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) { + String authHeader = this.dpsHeaders.getAuthorization(); + if (Strings.isNullOrEmpty(authHeader)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty"); + } + String user = this.dpsHeaders.getUserEmail(); + if (Strings.isNullOrEmpty(user)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty"); + } + return authHeader; + } else { + return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); + } + } +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java new file mode 100644 index 00000000..6da39c9e --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java @@ -0,0 +1,196 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.util; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.exceptions.JWTDecodeException; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.iam.v1.Iam; +import com.google.api.services.iam.v1.IamScopes; +import com.google.api.services.iam.v1.model.SignJwtRequest; +import com.google.api.services.iam.v1.model.SignJwtResponse; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.inject.Inject; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.IdToken; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; +import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { + + private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; + private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; + + private final JsonFactory JSON_FACTORY = new JacksonFactory(); + + private Iam iam; + + @Inject + private ITenantFactory tenantInfoServiceProvider; + @Inject + private IJwtCache cacheService; + @Inject + private JaxRsDpsLog log; + @Inject + private DpsHeaders dpsHeaders; + + @Value("${GOOGLE_AUDIENCES}") + public String GOOGLE_AUDIENCES; + + @Value("${INDEXER_HOST}") + public String INDEXER_HOST; + + public String getIdToken(String tenantName) { + this.log.info("Tenant name received for auth token is: " + tenantName); + TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); + if (tenant == null) { + this.log.error("Invalid tenant name receiving from pubsub"); + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub"); + } + try { + + IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); + // Add the user to DpsHeaders directly + this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); + + if (!IdToken.refreshToken(cachedToken)) { + return cachedToken.getTokenValue(); + } + + // Getting signed JWT + Map signJwtPayload = this.getJWTCreationPayload(tenant); + + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); + + String serviceAccountName = String + .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); + + Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); + + // Getting id token + List postParameters = new ArrayList<>(); + postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); + postParameters.add(new BasicNameValuePair("assertion", signedJwt)); + + HttpPost post = new HttpPost(JWT_AUDIENCE); + post.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); + post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); + + try (CloseableHttpClient httpclient = HttpClients.createDefault(); + CloseableHttpResponse httpResponse = httpclient.execute(post)) { + JsonObject jsonContent = new JsonParser().parse(EntityUtils.toString(httpResponse.getEntity())) + .getAsJsonObject(); + + if (!jsonContent.has("id_token")) { + log.error(String.format("Google IAM response: %s", jsonContent.toString())); + throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", + "The user is not authorized to perform this action"); + } + + String token = jsonContent.get("id_token").getAsString(); + IdToken idToken = IdToken.builder().tokenValue(token) + .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); + + this.cacheService.put(tenant.getServiceAccount(), idToken); + + return token; + } + } catch (JWTDecodeException e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", + "Invalid token, error decoding", e); + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", + e); + } + } + + public Iam getIam() throws Exception { + + if (this.iam == null) { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + + // Authenticate using Google Application Default Credentials. + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + List scopes = new ArrayList<>(); + // Enable full Cloud Platform scope. + scopes.add(IamScopes.CLOUD_PLATFORM); + credential = credential.createScoped(scopes); + } + + // Create IAM API object associated with the authenticated transport. + this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(INDEXER_HOST) + .build(); + } + + return this.iam; + } + + private Map getJWTCreationPayload(TenantInfo tenantInfo) { + + Map payload = new HashMap<>(); + String googleAudience = GOOGLE_AUDIENCES; + if (googleAudience.contains(",")) { + googleAudience = googleAudience.split(",")[0]; + } + payload.put("target_audience", googleAudience); + payload.put("exp", System.currentTimeMillis() / 1000 + 3600); + payload.put("iat", System.currentTimeMillis() / 1000); + payload.put("iss", tenantInfo.getServiceAccount()); + payload.put("aud", JWT_AUDIENCE); + + return payload; + } +} diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties new file mode 100644 index 00000000..09ea204d --- /dev/null +++ b/provider/indexer-reference/src/main/resources/application-dev.properties @@ -0,0 +1,40 @@ +# +# Copyright 2020 Google LLC +# Copyright 2020 EPAM Systems, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +GOOGLE_CLOUD_PROJECT=nice-etching-277309 + +INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com + +STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +STORAGE_RECORDS_BATCH_SIZE=20 + +INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue + +AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +CRS_API=example.com + +## use below values for gcp: opendes +REDIS_GROUP_HOST=127.0.0.1 +REDIS_SEARCH_HOST=127.0.0.1 + +GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + +mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties new file mode 100644 index 00000000..4d62d4ab --- /dev/null +++ b/provider/indexer-reference/src/main/resources/application.properties @@ -0,0 +1,78 @@ +# +# Copyright 2020 Google LLC +# Copyright 2020 EPAM Systems, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +LOG_PREFIX=indexer + +server.servlet.contextPath=/api/indexer/v2 +logging.level.org.springframework.web=DEBUG +server.port=8080 +JAVA_OPTS=-Xms3072m -Xmx3072m +JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 + +DEPLOYMENT_ENVIRONMENT=CLOUD + +REDIS_GROUP_PORT=6379 +REDIS_SEARCH_PORT=6379 +DEFAULT_DATA_COUNTRY=US + +#Default Cache Settings +SCHEMA_CACHE_EXPIRATION=60 +INDEX_CACHE_EXPIRATION=60 +ELASTIC_CACHE_EXPIRATION=1440 +CURSOR_CACHE_EXPIRATION=60 +# Kinds Cache expiration 2*24*60 +KINDS_CACHE_EXPIRATION=2880 +# Attributes Cache expiration 2*24*60 +ATTRIBUTES_CACHE_EXPIRATION=2880 + +KINDS_REDIS_DATABASE=1 +CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 +CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 + +GAE_SERVICE=indexer +KEY_RING=csqp +KMS_KEY=searchService + +ELASTIC_DATASTORE_KIND=SearchSettings +ELASTIC_DATASTORE_ID=indexer-service + +mongo.db.url=localhost:27017 +mongo.db.user= +mongo.db.password= + +INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com + +STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +STORAGE_RECORDS_BATCH_SIZE=20 + +INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue + +AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +CRS_API=example.com + +## use below values for gcp: opendes +REDIS_GROUP_HOST=127.0.0.1 +REDIS_SEARCH_HOST=127.0.0.1 + +GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + +mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 + diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java new file mode 100644 index 00000000..7ebfab8d --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.middleware; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.Collections; + +@RunWith(MockitoJUnitRunner.class) +public class IndexFilterTest { + + @InjectMocks + private IndexFilter indexFilter; + + @Mock + private DpsHeaders dpsHeaders; + + @Test + public void shouldSetCorrectResponseHeaders() throws IOException, ServletException { + HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class); + HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class); + FilterChain filterChain = Mockito.mock(FilterChain.class); + Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com"); + Mockito.when(httpServletRequest.getMethod()).thenReturn("POST"); + Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value"); + + indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); + + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString()); + Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString()); + Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString()); + Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString()); + Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString()); + Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString()); + Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString()); + Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString()); + Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value"); + Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse); + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java new file mode 100644 index 00000000..80e25bbb --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import com.google.common.collect.Lists; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.IndexInfo; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.mockito.Mockito.*; + + +@RunWith(SpringRunner.class) +@PrepareForTest({RestHighLevelClient.class}) +public class CronServiceImplTest { + + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private IRequestInfo requestInfo; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private CronServiceImpl sut; + + @InjectMocks + private DpsHeaders dpsHeaders; + + @Before + public void setup() { + + when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); + + ReflectionTestUtils.setField(this.sut, "CRON_INDEX_CLEANUP_THRESHOLD_DAYS", "3"); + ReflectionTestUtils.setField(this.sut, "CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS", "7"); + } + + @Test + public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception { + final String indexPattern = "tenant1-index-*"; + + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupIndices(indexPattern); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception); + + this.sut.cleanupIndices("tenant1-test-*"); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } + + @Test + public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test + public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_backend_throws_exception() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } +} \ No newline at end of file diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java new file mode 100644 index 00000000..e83cb648 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; +import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; +import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; + +@RunWith(SpringRunner.class) +public class ElasticSettingServiceTest { + + @Mock + private ITenantInfoService tenantInfoService; + @Mock + private IElasticRepository elasticRepository; + @Mock + private IElasticCredentialsCache elasticCredentialCache; + @Mock + private TenantInfo tenantInfo; + @InjectMocks + private ElasticSettingServiceImpl sut; + @Mock + private ClusterSettings clusterSettings; + @Mock + private DpsHeaders headersInfo; + + @Mock + private JaxRsDpsLog log; + + + public String GAE_SERVICE = "indexer"; + + private final String host = "db5c51c1.us-central1.gcp.cloud.es.io"; + private final int port = 9243; + private final String credentials = "name:password"; + + String cacheKey = ""; + + + @Before + public void setup() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); + when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); + sut.GAE_SERVICE = "indexer"; + clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build(); + cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); + } + + @Test + public void should_getValid_clusterSettings_fromCache() { + + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } + + @Test + public void should_getValid_clusterSettings_fromCosmosDB() { + + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings); + + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } + + @Test(expected = AppException.class) + public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException { + + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null); + + this.sut.getElasticClusterInformation(); + + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java new file mode 100644 index 00000000..9ef4a6ad --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java @@ -0,0 +1,193 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import org.apache.http.HttpEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.logging.AuditLogger; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; +import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(SpringRunner.class) +@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class}) +public class IndexCopyServiceImplTest { + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private HttpEntity httpEntity; + @Mock + private HttpEntity httpEntityRequest; + @Mock + private IRequestInfo requestInfo; + @Mock + private DpsHeaders headersInfo; + @Mock + private RestClient restClient; + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private IndexerMappingService mappingService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private Response response; + @Mock + private IElasticSettingService elasticSettingService; + @Mock + private AuditLogger auditLogger; + @Mock + private Map httpHeaders; + @InjectMocks + private IndexCopyServiceImpl sut; + + private ClusterSettings commonCluster; + + private Map correctMap; + + @Before + public void setup() { + + commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(response.getEntity()).thenReturn(httpEntity); + + Type mapType = new TypeToken>() {}.getType(); + String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}"; + correctMap = new Gson().fromJson(afterFormat, mapType); + + restHighLevelClient = mock(RestHighLevelClient.class); + + } + + @Test(expected = IOException.class) + public void should_throwIOException_when_indexMappingNotFound() throws Exception { + IOException exception = new IOException("Fail to get mapping for the given index from common cluster."); + + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception); + + this.sut.copyIndex("common:metadata:entity:1.0.0"); + } + + @Test(expected = IllegalArgumentException.class) + public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() { + try { + this.sut.copyIndex(null); + } catch (IOException e) { + fail("Should not throw IOException but illegalArgumentException."); + } + } + + @Test + public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() { + String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}"; + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + try { + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson); + Map resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test"); + Assert.assertEquals(resultMap, correctMap); + } catch (Exception ignored) { + } + } + + @Test + public void should_returnClusterInfo_getCommonClusterInformationtest() { + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + + when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster); + + String[] resultCommonCluster = this.sut.getCommonClusterInformation(); + Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]); + Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]); + Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); + } + } + + @Test(expected = AppException.class) + public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() { + try { + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + this.sut.createIndexInTenantCluster("test", "test", "test", correctMap); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); + } + } + + @Ignore + public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() { + //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest(). + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + Request request = new Request("POST", "/_reindex?wait_for_completion=false"); + request.setEntity(httpEntityRequest); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient); + when(restClient.performRequest(request)).thenReturn(response); + when(response.getEntity()).thenReturn(httpEntity); + Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster)); + } catch (IOException ignored) { + } + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java new file mode 100644 index 00000000..d091cf83 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -0,0 +1,341 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import com.google.gson.Gson; +import org.apache.http.StatusLine; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.action.bulk.BulkItemResponse.Failure; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.*; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; +import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.opengroup.osdu.indexer.util.TypeMapper; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +@Ignore +@RunWith(SpringRunner.class) +@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class, Config.class }) +public class IndexerMappingServiceTest { + + private final String kind = "tenant:test:test:1.0.0"; + private final String index = "tenant-test-test-1.0.0"; + private final String type = "test"; + private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}"; + + @Mock + private RestClient restClient; + @Mock + private Response response; + @Mock + private StatusLine statusLine; + + @InjectMocks + private IndexerMappingServiceImpl sut; + + @Mock + private ElasticClientHandler elasticClientHandler; + + @InjectMocks + private RestHighLevelClient restHighLevelClient; + + @InjectMocks + private IndexSchema indexSchema; + @InjectMocks + private IndicesClient indicesClient; + + @InjectMocks + private AcknowledgedResponse mappingResponse; + + @Before + public void setup() throws IOException { + initMocks(this); + mockStatic(Config.class); + when(Config.isPreDemo()).thenReturn(true); + Map dataMapping = new HashMap<>(); + dataMapping.put("Location", "geo_point"); + dataMapping.put("Msg", "text"); + Map metaMapping = new HashMap<>(); + metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); + metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL)); + this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping) + .build(); + + this.indicesClient = PowerMockito.mock(IndicesClient.class); + this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class); + + when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient); + when(this.restClient.performRequest(any())).thenReturn(response); + when(this.response.getStatusLine()).thenReturn(statusLine); + when(this.statusLine.getStatusCode()).thenReturn(200); + } + + @Test + public void should_returnValidMapping_givenFalseMerge_createMappingTest() { + try { + String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false); + assertEquals(mappingValid, mapping); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test + public void should_returnValidMapping_givenTrueMerge_createMappingTest() { + try { + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + + String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); + assertEquals(this.mappingValid, mapping); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test + public void should_returnValidMapping_givenExistType_createMappingTest() { + try { + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + + IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl()); + doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any()); + String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); + assertEquals(this.mappingValid, mapping); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test + public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception { + try { + Set indices = new HashSet(); + indices.add("indices 1"); + GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("any field", new HashMap()); + builder.endObject(); + BytesReference bytesReference = BytesReference.bytes(builder); + FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); + Map mapBuilder = new HashMap<>(); + mapBuilder.put("data.any field", mappingMetaData); + Map> mappingBuilder = new HashMap<>(); + mappingBuilder.put("any index 1", mapBuilder); + mappingBuilder.put("any index 2", mapBuilder); + Map>> mapping = new HashMap<>(); + mapping.put("indices 1", mappingBuilder); + when(getFieldMappingsResponse.mappings()).thenReturn(mapping); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + BulkByScrollResponse response = mock(BulkByScrollResponse.class); + doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); + when(response.getBulkFailures()).thenReturn(new ArrayList()); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + + this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field"); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test(expected = AppException.class) + public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception { + try { + Set indices = new HashSet(); + indices.add("invalid 1"); + GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("any field", new HashMap()); + builder.endObject(); + BytesReference bytesReference = BytesReference.bytes(builder); + FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); + Map mapBuilder = new HashMap<>(); + mapBuilder.put("data.any field", mappingMetaData); + Map> mappingBuilder = new HashMap<>(); + mappingBuilder.put("any index 1", mapBuilder); + mappingBuilder.put("any index 2", mapBuilder); + Map>> mapping = new HashMap<>(); + mapping.put("indices 1", mappingBuilder); + when(getFieldMappingsResponse.mappings()).thenReturn(mapping); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + BulkByScrollResponse response = mock(BulkByScrollResponse.class); + doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); + when(response.getBulkFailures()).thenReturn(new ArrayList()); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + + this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); + } catch (Exception e) { + throw e; + } + } + + @Test(expected = AppException.class) + public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception { + try { + Set indices = new HashSet(); + indices.add("indices 1"); + GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("any field", new HashMap()); + builder.endObject(); + BytesReference bytesReference = BytesReference.bytes(builder); + FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); + Map mapBuilder = new HashMap<>(); + mapBuilder.put("data.any field", mappingMetaData); + Map> mappingBuilder = new HashMap<>(); + mappingBuilder.put("any index 1", mapBuilder); + mappingBuilder.put("any index 2", mapBuilder); + Map>> mapping = new HashMap<>(); + mapping.put("indices 1", mappingBuilder); + when(getFieldMappingsResponse.mappings()).thenReturn(mapping); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + BulkByScrollResponse response = mock(BulkByScrollResponse.class); + doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); + when(response.getBulkFailures()).thenReturn(new ArrayList()); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid"); + } catch (Exception e) { + throw e; + } + } + + @Test(expected = AppException.class) + public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception { + try { + + Set indices = new HashSet(); + indices.add("indices 1"); + indices.add("indices Invalid"); + GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException("")); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("any field", new HashMap()); + builder.endObject(); + BytesReference bytesReference = BytesReference.bytes(builder); + FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); + Map mapBuilder = new HashMap<>(); + mapBuilder.put("data.any field", mappingMetaData); + Map> mappingBuilder = new HashMap<>(); + mappingBuilder.put("any index 1", mapBuilder); + mappingBuilder.put("any index 2", mapBuilder); + Map>> mapping = new HashMap<>(); + mapping.put("indices 1", mappingBuilder); + when(getFieldMappingsResponse.mappings()).thenReturn(mapping); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + BulkByScrollResponse response = mock(BulkByScrollResponse.class); + doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); + when(response.getBulkFailures()).thenReturn(new ArrayList()); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); + } catch (AppException e) { + throw e; + } + } + + @Test(expected = AppException.class) + public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() { + try { + Set indices = new HashSet(); + indices.add("indices 1"); + indices.add("indices Invalid"); + GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); + doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); + when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("any field", new HashMap()); + builder.endObject(); + BytesReference bytesReference = BytesReference.bytes(builder); + FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); + Map mapBuilder = new HashMap<>(); + mapBuilder.put("data.any field", mappingMetaData); + Map> mappingBuilder = new HashMap<>(); + mappingBuilder.put("any index 1", mapBuilder); + mappingBuilder.put("any index 2", mapBuilder); + Map>> mapping = new HashMap<>(); + mapping.put("indices 1", mappingBuilder); + when(getFieldMappingsResponse.mappings()).thenReturn(mapping); + doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); + BulkByScrollResponse response = mock(BulkByScrollResponse.class); + doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); + when(response.getBulkFailures()).thenReturn(new ArrayList()); + when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException("")); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); + } catch (AppException e) { + throw e; + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + + @Test + public void should_returnDocumentMapping_givenValidIndexSchema() { + + try { + Map documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema); + String documentMappingJson = new Gson().toJson(documentMapping); + assertEquals(this.mappingValid, documentMappingJson); + + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java new file mode 100644 index 00000000..e1dd9c64 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -0,0 +1,403 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import org.apache.http.HttpStatus; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.http.RequestStatus; +import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(SpringRunner.class) +@PrepareForTest({RestHighLevelClient.class}) +public class IndexerSchemaServiceTest { + + private final String kind = "tenant:test:test:1.0.0"; + private final String emptySchema = null; + private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}"; + + @Mock + private JaxRsDpsLog log; + @Mock + private StorageService storageService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private IndexerMappingService mappingService; + @Mock + private IndicesService indicesService; + @Mock + private ISchemaCache schemaCache; + @InjectMocks + private IndexSchemaServiceImpl sut; + + @Before + public void setup() { + initMocks(this); + RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + } + + @Test + public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception { + when(storageService.getStorageSchema(any())).thenReturn(emptySchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertNotNull(indexSchema); + } + + @Test + public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception { + when(storageService.getStorageSchema(any())).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception { + when(storageService.getStorageSchema(any())).thenReturn(someSchema); + when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { + try { + String invalidSchema = "{}}"; + when(storageService.getStorageSchema(any())).thenReturn(invalidSchema); + + this.sut.getIndexerInputSchema(kind, false); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); + Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); + } + } + + @Test + public void should_return_basic_schema_when_storage_returns_no_schema() { + IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false); + + assertNotNull(returnedSchema.getDataSchema()); + assertNotNull(returnedSchema); + assertEquals(kind, returnedSchema.getKind()); + } + + @Test + public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"endDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"type \"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"itemguid\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any()); + verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT); + assertEquals(e.getError().getMessage(), "error creating or merging index mapping"); + assertEquals(e.getError().getReason(), reason); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); + } + } + + @Test + public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN); + assertEquals(e.getError().getMessage(), "blah"); + assertEquals(e.getError().getReason(), reason); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); + } + } + + @Test + public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0")); + } + + @Test + public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaCache.get(kind)).thenReturn("schema"); + when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema"); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.schemaCache, times(2)).get(anyString()); + verify(this.schemaCache, times(2)).delete(anyString()); + } + + @Test + public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq(String.format("Kind: %s not found", kind))); + } + + @Test + public void should_sync_schema_with_storage() throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); + when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + + this.sut.syncIndexMappingWithStorageSchema(kind); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah")); + + try { + this.sut.syncIndexMappingWithStorageSchema(kind); + } catch (AppException e) { + assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT); + assertEquals(e.getError().getMessage(), "blah"); + assertEquals(e.getError().getReason(), "Index deletion error"); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); + } + + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any()); + } + + @Test + public void should_return_true_while_if_forceClean_requested() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true)); + } + + @Test + public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false)); + } + + @Test + public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false)); + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java new file mode 100644 index 00000000..f2a8886f --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java @@ -0,0 +1,354 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger +//// +//// Licensed under the Apache License, Version 2.0 (the "License"); +//// you may not use this file except in compliance with the License. +//// You may obtain a copy of the License at +//// +//// http://www.apache.org/licenses/LICENSE-2.0 +//// +//// Unless required by applicable law or agreed to in writing, software +//// distributed under the License is distributed on an "AS IS" BASIS, +//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//// See the License for the specific language governing permissions and +//// limitations under the License. +// +//package org.opendes.indexer.service; +// +//import com.google.gson.Gson; +//import com.google.gson.reflect.TypeToken; +//import org.elasticsearch.action.bulk.BulkItemResponse; +//import org.elasticsearch.action.bulk.BulkResponse; +//import org.elasticsearch.client.RequestOptions; +//import org.elasticsearch.client.RestHighLevelClient; +//import org.junit.Before; +//import org.junit.Ignore; +//import org.junit.Test; +//import org.junit.runner.RunWith; +//import org.mockito.InjectMocks; +//import org.mockito.Mock; +//import org.mockito.Spy; +//import org.opendes.client.api.DpsHeaders; +//import org.opendes.core.logging.JaxRsDpsLog; +//import org.opendes.core.model.DeploymentEnvironment; +//import org.opendes.core.model.RecordChangedMessages; +//import org.opendes.core.service.IndicesService; +//import org.opendes.core.util.Config; +//import org.opendes.core.util.ElasticClientHandler; +//import org.opendes.core.util.ElasticIndexNameResolver; +//import org.opendes.core.util.HeadersUtil; +//import org.opendes.indexer.logging.AuditLogger; +//import org.opendes.indexer.model.*; +//import org.opendes.indexer.publish.IPublisher; +//import org.opendes.indexer.util.IRequestInfo; +//import org.opendes.indexer.util.IndexerQueueTaskBuilder; +//import org.opendes.indexer.util.JobStatus; +//import org.opendes.indexer.util.RecordInfo; +//import org.powermock.core.classloader.annotations.PrepareForTest; +//import javax.inject.Inject; +//import org.springframework.context.annotation.Lazy; +//import org.springframework.test.context.junit4.SpringRunner; +// +//import java.io.IOException; +//import java.lang.reflect.Type; +//import java.util.*; +// +//import static java.util.Collections.singletonList; +//import static org.junit.Assert.*; +//import static org.mockito.Matchers.any; +//import static org.mockito.Mockito.verify; +//import static org.mockito.Mockito.when; +//import static org.powermock.api.mockito.PowerMockito.mock; +//import static org.powermock.api.mockito.PowerMockito.mockStatic; +// +//@Ignore +//@RunWith(SpringRunner.class) +//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class}) +//public class IndexerServiceTest { +// +// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," + +// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]"; +// private final String kind1 = "tenant1:testindexer1:well:1.0.0"; +// private final String kind2 = "tenant1:testindexer2:well:1.0.0"; +// private final String recordId1 = "tenant1:doc:test1"; +// private final String recordId2 = "tenant1:doc:test2"; +// private final String failureMassage = "test failure"; +// +// @Mock +// private IndexSchemaService indexSchemaService; +// @Mock +// private IndicesService indicesService; +// @Mock +// private IndexerMappingService indexerMappingService; +// @Mock +// private StorageService storageService; +// @Mock +// private IPublisher publisherImpl; +// @Mock +// private RestHighLevelClient restHighLevelClient; +// @Mock +// private ElasticClientHandler elasticClientHandler; +// @Mock +// private BulkResponse bulkResponse; +// @Mock +// private IRequestInfo requestInfo; +// @Mock +// private ElasticIndexNameResolver elasticIndexNameResolver; +// @Mock +// private AttributeParsingServiceImpl attributeParsingServiceImpl; +// @Mock +// private IndexerQueueTaskBuilder indexerQueueTaskBuilder; +// @Mock +// private JaxRsDpsLog log; +// @Mock +// private AuditLogger auditLogger; +// @InjectMocks +// private IndexerServiceImpl sut; +// @InjectMocks @Spy +// private JobStatus jobStatus = new JobStatus(); +// +// @Inject +// @Lazy +// private DpsHeaders dpsHeaders; +// private RecordChangedMessages recordChangedMessages; +// private List recordInfos; +// +// @Before +// public void setup() throws IOException { +// +// mockStatic(StorageAcl.class); +// mockStatic(Config.class); +// +// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); +// when(Config.getElasticClusterName()).thenReturn("CLUSTER"); +// when(Config.getElasticServerAddress()).thenReturn("testsite"); +// +// dpsHeaders = new DpsHeaders(); +// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1"); +// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); +// when(requestInfo.getHeaders()).thenReturn(dpsHeaders); +// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders()); +// +// Type listType = new TypeToken>() {}.getType(); +// recordInfos = (new Gson()).fromJson(pubsubMsg, listType); +// +// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); +// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse); +// +// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()}; +// when(bulkResponse.getItems()).thenReturn(responses); +// Map attr = new HashMap<>(); +// attr.put(DpsHeaders.ACCOUNT_ID, "slb"); +// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build(); +// when(StorageAcl.flattenAcl(any())).thenReturn(null); +// } +// +// @Test +// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception { +// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>()); +// +// assertNull(jobStatus); +// } +// +// @Test +// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() { +// try { +// indexSchemaServiceMock(kind1, null); +// indexSchemaServiceMock(kind2, null); +// List conversionStatus = new LinkedList<>(); +// List validRecords = new ArrayList<>(); +// Map storageData = new HashMap<>(); +// storageData.put("schema1", "test-value"); +// storageData.put("schema2", "test-value"); +// storageData.put("schema3", "test-value"); +// storageData.put("schema4", "test-value"); +// storageData.put("schema5", "test-value"); +// storageData.put("schema6", "test-value"); +// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); +// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); +// +// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); +// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); +// +// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); +// +// assertEquals(2, jobStatus.getStatusesList().size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); +// } catch (Exception e) { +// fail("Should not throw this exception" + e.getMessage()); +// } +// } +// +// @Test +// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() { +// try { +// indexSchemaServiceMock(kind1, null); +// indexSchemaServiceMock(kind2, null); +// List conversionStatuses = new LinkedList<>(); +// List status=new ArrayList<>(); +// status.add("crs bla bla"); +// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build(); +// conversionStatuses.add(conversionStatus); +// List validRecords = new ArrayList<>(); +// Map storageData = new HashMap<>(); +// storageData.put("schema1", "test-value"); +// storageData.put("schema2", "test-value"); +// storageData.put("schema3", "test-value"); +// storageData.put("schema4", "test-value"); +// storageData.put("schema5", "test-value"); +// storageData.put("schema6", "test-value"); +// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); +// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build(); +// +// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); +// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); +// +// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); +// +// assertEquals(2, jobStatus.getStatusesList().size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); +// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla")); +// } catch (Exception e) { +// fail("Should not throw this exception" + e.getMessage()); +// } +// } +// +// @Test +// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() { +// try { +// List validRecords = new ArrayList<>(); +// List conversionStatus = new LinkedList<>(); +// Map storageData = new HashMap<>(); +// storageData.put("schema1", "test-value"); +// storageData.put("schema2", "test-value"); +// storageData.put("schema3", "test-value"); +// storageData.put("schema4", "test-value"); +// storageData.put("schema5", "test-value"); +// storageData.put("schema6", "test-value"); +// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); +// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); +// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); +// +// Map schema = createSchema(); +// indexSchemaServiceMock(kind1, schema); +// indexSchemaServiceMock(kind2, null); +// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0"); +// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); +// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); +// +// assertEquals(2, jobStatus.getStatusesList().size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); +// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); +// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); +// } catch (Exception e) { +// fail("Should not throw this exception" + e.getMessage()); +// } +// } +// +// @Test +// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() { +// try { +// Map storageData = new HashMap<>(); +// storageData.put("schema1", "test-value"); +// List conversionStatus = new LinkedList<>(); +// List validRecords = new ArrayList<>(); +// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); +// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); +// +// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); +// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); +// Map schema = createSchema(); +// indexSchemaServiceMock(kind2, schema); +// indexSchemaServiceMock(kind1, null); +// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); +// +// assertEquals(2, jobStatus.getStatusesList().size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); +// } catch (Exception e) { +// fail("Should not throw this exception" + e.getMessage()); +// } +// } +// +// @Test +// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() { +// try { +// Map storageData = new HashMap<>(); +// List conversionStatus = new LinkedList<>(); +// +// storageData.put("schema1", "test-value"); +// List validRecords = new ArrayList<>(); +// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); +// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); +// +// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords); +// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); +// Map schema = createSchema(); +// indexSchemaServiceMock(kind2, schema); +// indexSchemaServiceMock(kind1, null); +// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); +// +// assertEquals(2, jobStatus.getStatusesList().size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); +// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); +// +// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)")); +// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)")); +// } catch (Exception e) { +// fail("Should not throw this exception" + e.getMessage()); +// } +// } +// +// private BulkItemResponse prepareResponseFail() { +// BulkItemResponse responseFail = mock(BulkItemResponse.class); +// when(responseFail.isFailed()).thenReturn(true); +// when(responseFail.getFailureMessage()).thenReturn(failureMassage); +// when(responseFail.getId()).thenReturn(recordId1); +// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure"))); +// return responseFail; +// } +// +// private BulkItemResponse prepareResponseSuccess() { +// BulkItemResponse responseSuccess = mock(BulkItemResponse.class); +// when(responseSuccess.getId()).thenReturn(recordId2); +// return responseSuccess; +// } +// +// private void indexSchemaServiceMock(String kind, Map schema) { +// if (schema == null) { +// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build(); +// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); +// } else { +// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build(); +// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); +// } +// } +// +// private Map createSchema() { +// Map schema = new HashMap<>(); +// schema.put("schema1", "keyword"); +// schema.put("schema2", "boolean"); +// schema.put("schema3", "date"); +// schema.put("schema6", "object"); +// return schema; +// } +//} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java new file mode 100644 index 00000000..502699c2 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; +import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.modules.junit4.PowerMockRunnerDelegate; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.*; + +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(PowerMockRunner.class) +@PowerMockRunnerDelegate(SpringRunner.class) +@PrepareForTest({Config.class}) +public class ReindexServiceTest { + + private final String cursor = "100"; + + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private StorageService storageService; + @Mock + private IRequestInfo requestInfo; + @Mock + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private ReindexServiceImpl sut; + + private RecordReindexRequest recordReindexRequest; + private RecordQueryResponse recordQueryResponse; + + private Map httpHeaders; + + @Before + public void setup() { + initMocks(this); + + mockStatic(UUID.class); + + recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build(); + recordQueryResponse = new RecordQueryResponse(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders); + when(requestInfo.getHeaders()).thenReturn(standardHeaders); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders); + } + + @Test + public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(null); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String response = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test + public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(new ArrayList<>()); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String response = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + @Ignore + @Test + public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setCursor(cursor); + List results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); + + mockStatic(Config.class); + when(Config.getStorageRecordsBatchSize()).thenReturn(1); + + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); + } + } + + @Test + public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() { + try { + List results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); + } + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java new file mode 100644 index 00000000..e8388a14 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java @@ -0,0 +1,251 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.service; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.core.common.model.indexer.RecordInfo; +import org.opengroup.osdu.core.common.model.http.HttpResponse; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.http.IUrlFetchService; +import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; +import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; +import org.opengroup.osdu.core.common.model.indexer.Records; +import org.springframework.http.HttpStatus; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.lang.reflect.Type; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(SpringRunner.class) +public class StorageServiceTest { + + @Mock + private IUrlFetchService urlFetchService; + @Mock + private JobStatus jobStatus; + @Mock + private JaxRsDpsLog log; + @Mock + private IRequestInfo requestInfo; + @InjectMocks + private StorageServiceImpl sut; + + private List ids; + private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465"; + private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813"; + + @Before + public void setup() { + + String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," + + "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]"; + + when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>()); + when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders()); + + Type listType = new TypeToken>() {}.getType(); + + List msgs = (new Gson()).fromJson(recordChangedMessages, listType); + jobStatus.initialize(msgs); + ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); + + ReflectionTestUtils.setField(this.sut, "STORAGE_RECORDS_BATCH_SIZE", "20"); + } + + @Test + public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException { + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(null); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + + should_return404_getValidStorageRecordsTest(); + } + + @Test + public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException { + + String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + + should_return404_getValidStorageRecordsTest(); + } + + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException { + + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); + + assertEquals(1, storageRecords.getRecords().size()); + } + + @Test + public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException { + + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); + + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813"); + } + + @Test + public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException { + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2)); + + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed")); + } + + @Test + public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception { + + RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build(); + + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class)); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + + RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest); + + assertEquals("100", recordQueryResponse.getCursor()); + assertNull(recordQueryResponse.getResults()); + } + + @Test + public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception { + + String validSchemaFromStorage = "{" + + " \"kind\": \"tenant:test:test:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"msg\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"references.entity\"," + + " \"kind\": \"string\"" + + " }" + + " ]," + + " \"ext\": null" + + "}"; + String kind = "tenant:test:test:1.0.0"; + + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.OK.value()); + httpResponse.setBody(validSchemaFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + + String recordSchemaResponse = this.sut.getStorageSchema(kind); + + assertNotNull(recordSchemaResponse); + } + + @Test + public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception { + + String kind = "tenant:test:test:1.0.0"; + + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value()); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + + String recordSchemaResponse = this.sut.getStorageSchema(kind); + + assertNull(recordSchemaResponse); + } + + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException { + + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); + + assertEquals(1, storageRecords.getRecords().size()); + + assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size()); + + assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0)); + } + + private void should_return404_getValidStorageRecordsTest() { + try { + this.sut.getStorageRecords(ids); + fail("Should throw exception"); + } catch (AppException e) { + assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } +} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java new file mode 100644 index 00000000..58dbef93 --- /dev/null +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java @@ -0,0 +1,171 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.util; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.services.iam.v1.Iam; +import com.google.api.services.iam.v1.model.SignJwtResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; +import org.opengroup.osdu.core.common.model.search.IdToken; +import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; +import org.opengroup.osdu.core.common.search.Config; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.when; + +@Ignore +@RunWith(SpringRunner.class) +@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, Config.class}) +public class ServiceAccountJwtGcpClientImplTest { + + private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw"; + + @Mock + private JaxRsDpsLog log; + @Mock + private GoogleCredential credential; + @Mock + private NetHttpTransport httpTransport; + @Mock + private SignJwtResponse signJwtResponse; + @Mock + private Iam iam; + @Mock + private Iam.Projects iamProject; + @Mock + private Iam.Projects.ServiceAccounts iamProjectServiceAccounts; + @Mock + private Iam.Projects.ServiceAccounts.SignJwt signJwt; + @Mock + private CloseableHttpClient httpClient; + @Mock + private CloseableHttpResponse httpResponse; +// @InjectMocks +// private TenantInfoServiceImpl tenantInfoServiceProvider; +// @Mock +// private TenantInfoServiceImpl tenantInfoService; + @Mock + private IJwtCache cacheService; + @InjectMocks @Spy + private ServiceAccountJwtGcpClientImpl sut; + @Before + public void setup() throws Exception { + initMocks(this); + +// mockStatic(GoogleNetHttpTransport.class); +// mockStatic(GoogleCredential.class); +// mockStatic(HttpClients.class); +// mockStatic(EntityUtils.class); +// mockStatic(Config.class); + + when(GoogleNetHttpTransport.newTrustedTransport()).thenReturn(httpTransport); + when(GoogleCredential.getApplicationDefault()).thenReturn(credential); + when(credential.createScopedRequired()).thenReturn(true); + when(credential.createScoped(any())).thenReturn(credential); + when(HttpClients.createDefault()).thenReturn(httpClient); + when(httpClient.execute(any())).thenReturn(httpResponse); + when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); + when(Config.getGoogleAudiences()).thenReturn("aud"); + +// when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService); + + TenantInfo tenantInfo = new TenantInfo(); + tenantInfo.setServiceAccount("tenant"); +// when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); + + when(this.sut.getIam()).thenReturn(iam); + when(this.iam.projects()).thenReturn(iamProject); + when(this.iamProject.serviceAccounts()).thenReturn(iamProjectServiceAccounts); + when(this.iamProjectServiceAccounts.signJwt(any(), any())).thenReturn(signJwt); + when(this.signJwt.execute()).thenReturn(signJwtResponse); + when(this.signJwtResponse.getSignedJwt()).thenReturn("testJwt"); + + } + + @Test + public void should_returnCachedToken_givenCachedToken_getIdTokenTest() { + String tokenValue = "tokenValue"; + IdToken idToken = IdToken.builder().tokenValue(tokenValue).expirationTimeMillis(System.currentTimeMillis() + 10000000L).build(); + when(this.cacheService.get(any())).thenReturn(idToken); + + String returnedIdToken = this.sut.getIdToken(tokenValue); + + Assert.assertEquals(tokenValue, returnedIdToken); + } + + @Test + public void should_returnValidToken_getIdTokenTest() throws Exception { + when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", JWT_TOKEN)); + + String returnedToken = this.sut.getIdToken("tenant"); + + Assert.assertEquals(JWT_TOKEN, returnedToken); + } + + @Test + public void should_return500_given_invalidJWTResponse_getIdTokenException() { + try { + when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", "invalid jwt")); + + this.sut.getIdToken("tenant"); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); + Assert.assertEquals("Invalid token, error decoding", e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + @Test + public void should_return403_given_missingIdTokenResponse_getIdTokenException() { + try { + when(EntityUtils.toString(any())).thenReturn("{}"); + + this.sut.getIdToken("tenant"); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); + Assert.assertEquals("The user is not authorized to perform this action", e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } +} diff --git a/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000..ca6ee9ce --- /dev/null +++ b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file -- GitLab From 8ec995b2b0c5394ab972492f6bfcf0336acf9b3f Mon Sep 17 00:00:00 2001 From: Aliaksandr Ramanovich1 Date: Wed, 23 Dec 2020 06:12:02 +0000 Subject: [PATCH 03/11] Parametrize cloudbuild.yml --- provider/indexer-gcp/cloudbuild/cloudbuild.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml index a0839623..601ba2e8 100644 --- a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml +++ b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml @@ -17,15 +17,14 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: [ - 'build', - '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', - '--build-arg', 'PORT=${_PORT}', - '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}', - '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest', - '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', - '.' + 'build', + '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', + '--build-arg', 'PORT=${_PORT}', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest', + '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', + '.' ] images: - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}' - -- GitLab From fb3b518d537d4f8f6e93da37d7d1bdc4faeea60c Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Tue, 29 Dec 2020 08:30:46 +0000 Subject: [PATCH 04/11] Update cloudbuild.yaml --- provider/indexer-gcp/cloudbuild/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml index 601ba2e8..1d0c9ba8 100644 --- a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml +++ b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml @@ -28,3 +28,4 @@ steps: images: - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}' + -- GitLab From 039d2f510188804bc91c040e82014849a815a163 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Mon, 11 Jan 2021 11:58:24 +0400 Subject: [PATCH 05/11] Removed an unused sut.GAE_SERVICE from ElasticSettingServiceTest --- .../osdu/indexer/service/ElasticSettingServiceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java index e83cb648..9fcf3b4a 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -72,7 +72,6 @@ public class ElasticSettingServiceTest { when(tenantInfo.getName()).thenReturn("tenant1"); when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); - sut.GAE_SERVICE = "indexer"; clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build(); cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); } -- GitLab From 4ee047237d23d315c98fcd23b4ccafa154436520 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Mon, 11 Jan 2021 12:38:48 +0400 Subject: [PATCH 06/11] Fixed sonar comments for reference provider --- .../indexer/service/CronServiceImplTest.java | 8 +- .../service/ElasticSettingServiceTest.java | 4 + .../service/IndexerMappingServiceTest.java | 10 +- .../service/IndexerSchemaServiceTest.java | 41 +- .../indexer/service/IndexerServiceTest.java | 354 ------------------ .../indexer/service/ReindexServiceTest.java | 9 +- .../indexer/service/StorageServiceTest.java | 6 +- .../ServiceAccountJwtGcpClientImplTest.java | 10 +- 8 files changed, 49 insertions(+), 393 deletions(-) delete mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java index 80e25bbb..9633d128 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -29,10 +29,10 @@ import org.opengroup.osdu.core.common.model.search.IndexInfo; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.util.ReflectionTestUtils; import java.io.IOException; import java.time.Instant; @@ -52,6 +52,8 @@ public class CronServiceImplTest { @Mock private ElasticClientHandler elasticClientHandler; @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock private IRequestInfo requestInfo; @Mock private JaxRsDpsLog log; @@ -66,8 +68,8 @@ public class CronServiceImplTest { when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); - ReflectionTestUtils.setField(this.sut, "CRON_INDEX_CLEANUP_THRESHOLD_DAYS", "3"); - ReflectionTestUtils.setField(this.sut, "CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS", "7"); + when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3); + when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3); } @Test diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java index 9fcf3b4a..f9645a50 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -30,6 +30,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.test.context.junit4.SpringRunner; import static org.junit.Assert.assertEquals; @@ -46,6 +47,8 @@ public class ElasticSettingServiceTest { @Mock private IElasticCredentialsCache elasticCredentialCache; @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock private TenantInfo tenantInfo; @InjectMocks private ElasticSettingServiceImpl sut; @@ -72,6 +75,7 @@ public class ElasticSettingServiceTest { when(tenantInfo.getName()).thenReturn("tenant1"); when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); + when(configurationProperties.getGaeService()).thenReturn("indexer"); clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build(); cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java index d091cf83..b4eca3a4 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -38,7 +38,7 @@ import org.mockito.Mock; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.powermock.api.mockito.PowerMockito; @@ -54,12 +54,11 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.powermock.api.mockito.PowerMockito.when; @Ignore @RunWith(SpringRunner.class) -@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class, Config.class }) +@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class}) public class IndexerMappingServiceTest { private final String kind = "tenant:test:test:1.0.0"; @@ -67,6 +66,8 @@ public class IndexerMappingServiceTest { private final String type = "test"; private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}"; + @Mock + private IndexerConfigurationProperties configurationProperties; @Mock private RestClient restClient; @Mock @@ -94,8 +95,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { initMocks(this); - mockStatic(Config.class); - when(Config.isPreDemo()).thenReturn(true); + when(configurationProperties.isPreDemo()).thenReturn(true); Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); dataMapping.put("Msg", "text"); diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java index e1dd9c64..27b5f786 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -32,6 +32,7 @@ import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.http.RequestStatus; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl; import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -59,7 +60,7 @@ public class IndexerSchemaServiceTest { @Mock private JaxRsDpsLog log; @Mock - private StorageService storageService; + private SchemaProviderImpl schemaProvider; @Mock private ElasticClientHandler elasticClientHandler; @Mock @@ -82,7 +83,7 @@ public class IndexerSchemaServiceTest { @Test public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception { - when(storageService.getStorageSchema(any())).thenReturn(emptySchema); + when(schemaProvider.getSchema(any())).thenReturn(emptySchema); IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); @@ -91,7 +92,7 @@ public class IndexerSchemaServiceTest { @Test public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception { - when(storageService.getStorageSchema(any())).thenReturn(someSchema); + when(schemaProvider.getSchema(any())).thenReturn(someSchema); IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); @@ -100,7 +101,7 @@ public class IndexerSchemaServiceTest { @Test public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception { - when(storageService.getStorageSchema(any())).thenReturn(someSchema); + when(schemaProvider.getSchema(any())).thenReturn(someSchema); when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); @@ -112,7 +113,7 @@ public class IndexerSchemaServiceTest { public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { try { String invalidSchema = "{}}"; - when(storageService.getStorageSchema(any())).thenReturn(invalidSchema); + when(schemaProvider.getSchema(any())).thenReturn(invalidSchema); this.sut.getIndexerInputSchema(kind, false); fail("Should throw exception"); @@ -167,7 +168,7 @@ public class IndexerSchemaServiceTest { when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); this.sut.processSchemaMessages(schemaMessages); @@ -198,7 +199,7 @@ public class IndexerSchemaServiceTest { when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); this.sut.processSchemaMessages(schemaMessages); @@ -226,15 +227,15 @@ public class IndexerSchemaServiceTest { when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); try { this.sut.processSchemaMessages(schemaMessages); } catch (AppException e) { - assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT); - assertEquals(e.getError().getMessage(), "error creating or merging index mapping"); - assertEquals(e.getError().getReason(), reason); + assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode()); + assertEquals("error creating or merging index mapping", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); } catch (Exception e) { fail("Should not throw this exception " + e.getMessage()); } @@ -259,15 +260,15 @@ public class IndexerSchemaServiceTest { when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); try { this.sut.processSchemaMessages(schemaMessages); } catch (AppException e) { - assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN); - assertEquals(e.getError().getMessage(), "blah"); - assertEquals(e.getError().getReason(), reason); + assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); } catch (Exception e) { fail("Should not throw this exception " + e.getMessage()); } @@ -285,7 +286,7 @@ public class IndexerSchemaServiceTest { when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); this.sut.processSchemaMessages(schemaMessages); @@ -339,7 +340,7 @@ public class IndexerSchemaServiceTest { when(this.schemaCache.get(kind)).thenReturn(null); when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); - when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); this.sut.syncIndexMappingWithStorageSchema(kind); @@ -361,9 +362,9 @@ public class IndexerSchemaServiceTest { try { this.sut.syncIndexMappingWithStorageSchema(kind); } catch (AppException e) { - assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT); - assertEquals(e.getError().getMessage(), "blah"); - assertEquals(e.getError().getReason(), "Index deletion error"); + assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals("Index deletion error", e.getError().getReason()); } catch (Exception e) { fail("Should not throw this exception " + e.getMessage()); } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java deleted file mode 100644 index f2a8886f..00000000 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger -//// -//// Licensed under the Apache License, Version 2.0 (the "License"); -//// you may not use this file except in compliance with the License. -//// You may obtain a copy of the License at -//// -//// http://www.apache.org/licenses/LICENSE-2.0 -//// -//// Unless required by applicable law or agreed to in writing, software -//// distributed under the License is distributed on an "AS IS" BASIS, -//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//// See the License for the specific language governing permissions and -//// limitations under the License. -// -//package org.opendes.indexer.service; -// -//import com.google.gson.Gson; -//import com.google.gson.reflect.TypeToken; -//import org.elasticsearch.action.bulk.BulkItemResponse; -//import org.elasticsearch.action.bulk.BulkResponse; -//import org.elasticsearch.client.RequestOptions; -//import org.elasticsearch.client.RestHighLevelClient; -//import org.junit.Before; -//import org.junit.Ignore; -//import org.junit.Test; -//import org.junit.runner.RunWith; -//import org.mockito.InjectMocks; -//import org.mockito.Mock; -//import org.mockito.Spy; -//import org.opendes.client.api.DpsHeaders; -//import org.opendes.core.logging.JaxRsDpsLog; -//import org.opendes.core.model.DeploymentEnvironment; -//import org.opendes.core.model.RecordChangedMessages; -//import org.opendes.core.service.IndicesService; -//import org.opendes.core.util.Config; -//import org.opendes.core.util.ElasticClientHandler; -//import org.opendes.core.util.ElasticIndexNameResolver; -//import org.opendes.core.util.HeadersUtil; -//import org.opendes.indexer.logging.AuditLogger; -//import org.opendes.indexer.model.*; -//import org.opendes.indexer.publish.IPublisher; -//import org.opendes.indexer.util.IRequestInfo; -//import org.opendes.indexer.util.IndexerQueueTaskBuilder; -//import org.opendes.indexer.util.JobStatus; -//import org.opendes.indexer.util.RecordInfo; -//import org.powermock.core.classloader.annotations.PrepareForTest; -//import javax.inject.Inject; -//import org.springframework.context.annotation.Lazy; -//import org.springframework.test.context.junit4.SpringRunner; -// -//import java.io.IOException; -//import java.lang.reflect.Type; -//import java.util.*; -// -//import static java.util.Collections.singletonList; -//import static org.junit.Assert.*; -//import static org.mockito.Matchers.any; -//import static org.mockito.Mockito.verify; -//import static org.mockito.Mockito.when; -//import static org.powermock.api.mockito.PowerMockito.mock; -//import static org.powermock.api.mockito.PowerMockito.mockStatic; -// -//@Ignore -//@RunWith(SpringRunner.class) -//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class}) -//public class IndexerServiceTest { -// -// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," + -// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]"; -// private final String kind1 = "tenant1:testindexer1:well:1.0.0"; -// private final String kind2 = "tenant1:testindexer2:well:1.0.0"; -// private final String recordId1 = "tenant1:doc:test1"; -// private final String recordId2 = "tenant1:doc:test2"; -// private final String failureMassage = "test failure"; -// -// @Mock -// private IndexSchemaService indexSchemaService; -// @Mock -// private IndicesService indicesService; -// @Mock -// private IndexerMappingService indexerMappingService; -// @Mock -// private StorageService storageService; -// @Mock -// private IPublisher publisherImpl; -// @Mock -// private RestHighLevelClient restHighLevelClient; -// @Mock -// private ElasticClientHandler elasticClientHandler; -// @Mock -// private BulkResponse bulkResponse; -// @Mock -// private IRequestInfo requestInfo; -// @Mock -// private ElasticIndexNameResolver elasticIndexNameResolver; -// @Mock -// private AttributeParsingServiceImpl attributeParsingServiceImpl; -// @Mock -// private IndexerQueueTaskBuilder indexerQueueTaskBuilder; -// @Mock -// private JaxRsDpsLog log; -// @Mock -// private AuditLogger auditLogger; -// @InjectMocks -// private IndexerServiceImpl sut; -// @InjectMocks @Spy -// private JobStatus jobStatus = new JobStatus(); -// -// @Inject -// @Lazy -// private DpsHeaders dpsHeaders; -// private RecordChangedMessages recordChangedMessages; -// private List recordInfos; -// -// @Before -// public void setup() throws IOException { -// -// mockStatic(StorageAcl.class); -// mockStatic(Config.class); -// -// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); -// when(Config.getElasticClusterName()).thenReturn("CLUSTER"); -// when(Config.getElasticServerAddress()).thenReturn("testsite"); -// -// dpsHeaders = new DpsHeaders(); -// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1"); -// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); -// when(requestInfo.getHeaders()).thenReturn(dpsHeaders); -// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders()); -// -// Type listType = new TypeToken>() {}.getType(); -// recordInfos = (new Gson()).fromJson(pubsubMsg, listType); -// -// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); -// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse); -// -// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()}; -// when(bulkResponse.getItems()).thenReturn(responses); -// Map attr = new HashMap<>(); -// attr.put(DpsHeaders.ACCOUNT_ID, "slb"); -// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build(); -// when(StorageAcl.flattenAcl(any())).thenReturn(null); -// } -// -// @Test -// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception { -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>()); -// -// assertNull(jobStatus); -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List conversionStatus = new LinkedList<>(); -// List validRecords = new ArrayList<>(); -// Map storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() { -// try { -// indexSchemaServiceMock(kind1, null); -// indexSchemaServiceMock(kind2, null); -// List conversionStatuses = new LinkedList<>(); -// List status=new ArrayList<>(); -// status.add("crs bla bla"); -// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build(); -// conversionStatuses.add(conversionStatus); -// List validRecords = new ArrayList<>(); -// Map storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() { -// try { -// List validRecords = new ArrayList<>(); -// List conversionStatus = new LinkedList<>(); -// Map storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// storageData.put("schema2", "test-value"); -// storageData.put("schema3", "test-value"); -// storageData.put("schema4", "test-value"); -// storageData.put("schema5", "test-value"); -// storageData.put("schema6", "test-value"); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// -// Map schema = createSchema(); -// indexSchemaServiceMock(kind1, schema); -// indexSchemaServiceMock(kind2, null); -// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0"); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size()); -// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() { -// try { -// Map storageData = new HashMap<>(); -// storageData.put("schema1", "test-value"); -// List conversionStatus = new LinkedList<>(); -// List validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// @Test -// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() { -// try { -// Map storageData = new HashMap<>(); -// List conversionStatus = new LinkedList<>(); -// -// storageData.put("schema1", "test-value"); -// List validRecords = new ArrayList<>(); -// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build()); -// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build(); -// -// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords); -// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true); -// Map schema = createSchema(); -// indexSchemaServiceMock(kind2, schema); -// indexSchemaServiceMock(kind1, null); -// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos); -// -// assertEquals(2, jobStatus.getStatusesList().size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size()); -// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size()); -// -// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)")); -// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)")); -// } catch (Exception e) { -// fail("Should not throw this exception" + e.getMessage()); -// } -// } -// -// private BulkItemResponse prepareResponseFail() { -// BulkItemResponse responseFail = mock(BulkItemResponse.class); -// when(responseFail.isFailed()).thenReturn(true); -// when(responseFail.getFailureMessage()).thenReturn(failureMassage); -// when(responseFail.getId()).thenReturn(recordId1); -// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure"))); -// return responseFail; -// } -// -// private BulkItemResponse prepareResponseSuccess() { -// BulkItemResponse responseSuccess = mock(BulkItemResponse.class); -// when(responseSuccess.getId()).thenReturn(recordId2); -// return responseSuccess; -// } -// -// private void indexSchemaServiceMock(String kind, Map schema) { -// if (schema == null) { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } else { -// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build(); -// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema); -// } -// } -// -// private Map createSchema() { -// Map schema = new HashMap<>(); -// schema.put("schema1", "keyword"); -// schema.put("schema2", "boolean"); -// schema.put("schema3", "date"); -// schema.put("schema6", "object"); -// return schema; -// } -//} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java index 502699c2..8cc2f75c 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java @@ -29,9 +29,8 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; import org.springframework.test.context.junit4.SpringRunner; @@ -46,13 +45,14 @@ import static org.powermock.api.mockito.PowerMockito.when; @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(SpringRunner.class) -@PrepareForTest({Config.class}) public class ReindexServiceTest { private final String cursor = "100"; private final String correlationId = UUID.randomUUID().toString(); + @Mock + private IndexerConfigurationProperties configurationProperties; @Mock private StorageService storageService; @Mock @@ -123,8 +123,7 @@ public class ReindexServiceTest { results.add("test1"); recordQueryResponse.setResults(results); - mockStatic(Config.class); - when(Config.getStorageRecordsBatchSize()).thenReturn(1); + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1); when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java index e8388a14..b4ea76b8 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java @@ -39,9 +39,9 @@ import org.opengroup.osdu.core.common.http.IUrlFetchService; import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; import org.opengroup.osdu.core.common.model.indexer.Records; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.http.HttpStatus; import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.util.ReflectionTestUtils; import java.lang.reflect.Type; import java.net.URISyntaxException; @@ -67,6 +67,8 @@ public class StorageServiceTest { private JaxRsDpsLog log; @Mock private IRequestInfo requestInfo; + @Mock + private IndexerConfigurationProperties configurationProperties; @InjectMocks private StorageServiceImpl sut; @@ -89,7 +91,7 @@ public class StorageServiceTest { jobStatus.initialize(msgs); ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); - ReflectionTestUtils.setField(this.sut, "STORAGE_RECORDS_BATCH_SIZE", "20"); + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20); } @Test diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java index 58dbef93..2c681985 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java @@ -41,7 +41,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.model.search.IdToken; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; @@ -52,11 +52,13 @@ import static org.powermock.api.mockito.PowerMockito.when; @Ignore @RunWith(SpringRunner.class) -@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, Config.class}) +@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, IndexerConfigurationProperties.class}) public class ServiceAccountJwtGcpClientImplTest { private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw"; + @Mock + private IndexerConfigurationProperties indexerConfigurationProperties; @Mock private JaxRsDpsLog log; @Mock @@ -101,8 +103,8 @@ public class ServiceAccountJwtGcpClientImplTest { when(credential.createScoped(any())).thenReturn(credential); when(HttpClients.createDefault()).thenReturn(httpClient); when(httpClient.execute(any())).thenReturn(httpResponse); - when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); - when(Config.getGoogleAudiences()).thenReturn("aud"); + when(indexerConfigurationProperties.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); + when(indexerConfigurationProperties.getGoogleAudiences()).thenReturn("aud"); // when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService); -- GitLab From 314d225a788ef9a2df4c1e599611519734aecea0 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Mon, 18 Jan 2021 14:10:45 +0400 Subject: [PATCH 07/11] Refactoring code. --- .../indexer/IndexerAnthosApplication.java | 15 +- .../osdu/indexer/ServletInitializer.java | 13 +- .../osdu/indexer/cache/AttributesCache.java | 82 +- .../cache/DatastoreCredentialCache.java | 19 +- .../cache/ElasticCredentialsCache.java | 80 +- .../osdu/indexer/cache/IndexCache.java | 76 +- .../osdu/indexer/cache/JwtCache.java | 78 +- .../osdu/indexer/cache/KindsCache.java | 70 +- .../osdu/indexer/cache/SchemaCache.java | 76 +- .../config/EntitlementsConfigProperties.java | 32 + .../config/MongoDBConfigProperties.java | 35 + .../config/RabbitMqConfigProperties.java | 32 + .../indexer/di/EntitlementsClientFactory.java | 47 +- .../osdu/indexer/di/RabbitMQFactoryImpl.java | 81 +- .../osdu/indexer/di/TenantFactoryImpl.java | 113 +-- .../opengroup/osdu/indexer/kms/KmsClient.java | 105 -- .../indexer/messagebus/IMessageFactory.java | 4 +- .../osdu/indexer/middleware/IndexFilter.java | 160 ++-- .../persistence/DatastoreCredential.java | 120 +-- .../indexer/persistence/DatastoreFactory.java | 73 +- .../persistence/ElasticRepositoryMongoDB.java | 81 +- .../indexer/persistence/MongoDdmsClient.java | 19 +- .../osdu/indexer/publish/PublisherImpl.java | 124 +-- .../security/GSuiteSecurityConfig.java | 28 +- .../indexer/service/IndexerServiceImpl.java | 901 ++++++++++-------- .../indexer/util/DpsHeaderFactoryGcp.java | 39 +- .../osdu/indexer/util/MongoClientHandler.java | 25 +- .../osdu/indexer/util/RequestInfoImpl.java | 167 ++-- .../util/ServiceAccountJwtGcpClientImpl.java | 276 +++--- .../main/resources/application-dev.properties | 32 +- .../src/main/resources/application.properties | 80 +- .../indexer/middleware/IndexFilterTest.java | 85 +- .../indexer/service/CronServiceImplTest.java | 214 +++-- .../service/ElasticSettingServiceTest.java | 133 +-- .../service/IndexCopyServiceImplTest.java | 290 +++--- .../service/IndexerMappingServiceTest.java | 341 ------- .../service/IndexerSchemaServiceTest.java | 763 ++++++++------- .../indexer/service/ReindexServiceTest.java | 210 ++-- .../indexer/service/StorageServiceTest.java | 339 +++---- .../ServiceAccountJwtGcpClientImplTest.java | 173 ---- 40 files changed, 2684 insertions(+), 2947 deletions(-) create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java create mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java delete mode 100644 provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java delete mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java delete mode 100644 provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java index a87c9d45..d2c3e68f 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,14 +27,15 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.FilterType; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) +@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, + ManagementWebSecurityAutoConfiguration.class}) @Configuration @ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = { - @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)}) public class IndexerAnthosApplication { - public static void main(String[] args) { - SpringApplication.run(IndexerAnthosApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(IndexerAnthosApplication.class, args); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java index ccc93114..ec6996fe 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +21,10 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; public class ServletInitializer extends SpringBootServletInitializer { - @Override - protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { - return application.sources(IndexerAnthosApplication.class); - } + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(IndexerAnthosApplication.class); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java index bdfa51cd..b60b4d14 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,47 +17,49 @@ package org.opengroup.osdu.indexer.cache; +import java.util.Set; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Set; @Component -public class AttributesCache implements IAttributesCache, AutoCloseable { - - private RedisCache cache; - - public AttributesCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { - - cache = new RedisCache(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); - } - - @Override - public void put(String key, Set value) { - this.cache.put(key, value); - } - - @Override - public Set get(String key) { - return this.cache.get(key); - } - - @Override - public void delete(String key) { - this.cache.delete(key); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } - - @Override - public void close() { - this.cache.close(); - } +public class AttributesCache implements IAttributesCache, AutoCloseable { + + private RedisCache cache; + + @Autowired + public AttributesCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + Boolean.class); + } + + @Override + public void put(String key, Set value) { + this.cache.put(key, value); + } + + @Override + public Set get(String key) { + return this.cache.get(key); + } + + @Override + public void delete(String key) { + this.cache.delete(key); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } + + @Override + public void close() { + this.cache.close(); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java index d17e26f7..6e832b44 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,14 +19,19 @@ package org.opengroup.osdu.indexer.cache; import com.google.auth.oauth2.AccessToken; import org.opengroup.osdu.core.common.cache.RedisCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class DatastoreCredentialCache extends RedisCache { - // Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration - public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { - super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class); - } + @Autowired + public DatastoreCredentialCache(IndexerConfigurationProperties indexerConfigurationProperties) { + super(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + 58 * 60, + String.class, + AccessToken.class); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java index 7a5d5f6f..24804bc9 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,43 +20,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class ElasticCredentialsCache implements IElasticCredentialsCache, AutoCloseable { - - private RedisCache cache; - - public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, String.class, ClusterSettings.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, ClusterSettings o) { - this.cache.put(s,o); - } - - @Override - public ClusterSettings get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } +public class ElasticCredentialsCache implements IElasticCredentialsCache, + AutoCloseable { + + private RedisCache cache; + + @Autowired + public ElasticCredentialsCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + ClusterSettings.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, ClusterSettings o) { + this.cache.put(s, o); + } + + @Override + public ClusterSettings get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java index d9b7197f..de447717 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,42 +19,46 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class IndexCache implements IIndexCache, AutoCloseable { - private RedisCache cache; - - public IndexCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, Boolean o) { - this.cache.put(s, o); - } - - @Override - public Boolean get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + private RedisCache cache; + + @Autowired + public IndexCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getIndexCacheExpiration() * 60, + String.class, + Boolean.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, Boolean o) { + this.cache.put(s, o); + } + + @Override + public Boolean get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java index 2b442b62..9a2a02d7 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,43 +20,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.search.IdToken; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.stereotype.Component; @Component public class JwtCache implements IJwtCache, AutoCloseable { - RedisCache cache; - - // google service account id_token can be requested only for 1 hr - private final static int EXPIRED_AFTER = 59; - - public JwtCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - EXPIRED_AFTER * 60, String.class, IdToken.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, IdToken o) { - this.cache.put(s, o); - } - - @Override - public IdToken get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + RedisCache cache; + + // google service account id_token can be requested only for 1 hr + private final static int EXPIRED_AFTER = 59; + + public JwtCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + EXPIRED_AFTER * 60, + String.class, + IdToken.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, IdToken o) { + this.cache.put(s, o); + } + + @Override + public IdToken get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java index d70210a0..afc2a8c3 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,48 +17,50 @@ package org.opengroup.osdu.indexer.cache; +import java.util.Set; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Set; - @Component public class KindsCache implements IKindsCache, AutoCloseable { - private RedisCache cache; - public KindsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION, - @Value("${KINDS_REDIS_DATABASE}") final String KINDS_REDIS_DATABASE) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60, - Integer.parseInt(KINDS_REDIS_DATABASE), String.class, Set.class); - } + private RedisCache cache; + + @Autowired + public KindsCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getKindsCacheExpiration() * 60, + indexerConfigurationProperties.getKindsRedisDatabase(), + String.class, + Set.class); + } - @Override - public void close() throws Exception { - this.cache.close(); - } + @Override + public void close() throws Exception { + this.cache.close(); + } - @Override - public void put(String s, Set o) { - this.cache.put(s, o); - } + @Override + public void put(String s, Set o) { + this.cache.put(s, o); + } - @Override - public Set get(String s) { - return this.cache.get(s); - } + @Override + public Set get(String s) { + return this.cache.get(s); + } - @Override - public void delete(String s) { - this.cache.delete(s); - } + @Override + public void delete(String s) { + this.cache.delete(s); + } - @Override - public void clearAll() { - this.cache.clearAll(); - } + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java index 1fc8b411..3d84e229 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,43 +18,47 @@ package org.opengroup.osdu.indexer.cache; import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class SchemaCache implements ISchemaCache, AutoCloseable { - private RedisCache cache; - - public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, - @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT, - @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), - Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); - } - - @Override - public void close() throws Exception { - this.cache.close(); - } - - @Override - public void put(String s, String o) { - this.cache.put(s, o); - } - - @Override - public String get(String s) { - return this.cache.get(s); - } - - @Override - public void delete(String s) { - this.cache.delete(s); - } - - @Override - public void clearAll() { - this.cache.clearAll(); - } + + private RedisCache cache; + + @Autowired + public SchemaCache(IndexerConfigurationProperties indexerConfigurationProperties) { + cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(), + Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()), + indexerConfigurationProperties.getSchemaCacheExpiration() * 60, + String.class, + String.class); + } + + @Override + public void close() throws Exception { + this.cache.close(); + } + + @Override + public void put(String s, String o) { + this.cache.put(s, o); + } + + @Override + public String get(String s) { + return this.cache.get(s); + } + + @Override + public void delete(String s) { + this.cache.delete(s); + } + + @Override + public void clearAll() { + this.cache.clearAll(); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java new file mode 100644 index 00000000..1ba4425a --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class EntitlementsConfigProperties { + + private String AuthorizeApi; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java new file mode 100644 index 00000000..080f76df --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MongoDBConfigProperties { + + private String mongoDbUrl; + private String mongoDbUser; + private String mongoDbPassword; + private String mongoDbName; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java new file mode 100644 index 00000000..55efbf1a --- /dev/null +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.indexer.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RabbitMqConfigProperties { + + private String mbRabbitMqUri; +} diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java index 8460aa5f..45131176 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,8 @@ package org.opengroup.osdu.indexer.di; import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig; import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory; import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.AbstractFactoryBean; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -31,24 +32,24 @@ import org.springframework.web.context.annotation.RequestScope; @Lazy public class EntitlementsClientFactory extends AbstractFactoryBean { - @Value("${AUTHORIZE_API}") - private String AUTHORIZE_API; - - @Value("${AUTHORIZE_API_KEY:}") - private String AUTHORIZE_API_KEY; - - @Override - protected IEntitlementsFactory createInstance() throws Exception { - - return new EntitlementsFactory(EntitlementsAPIConfig - .builder() - .rootUrl(AUTHORIZE_API) - .apiKey(AUTHORIZE_API_KEY) - .build()); - } - - @Override - public Class getObjectType() { - return IEntitlementsFactory.class; - } + private EntitlementsConfigProperties entitlementsConfigProperties; + + @Override + protected IEntitlementsFactory createInstance() throws Exception { + return new EntitlementsFactory(EntitlementsAPIConfig + .builder() + .rootUrl(entitlementsConfigProperties.getAuthorizeApi()) + .build()); + } + + @Override + public Class getObjectType() { + return IEntitlementsFactory.class; + } + + @Autowired + public void setEntitlementsConfigProperties( + EntitlementsConfigProperties entitlementsConfigProperties) { + this.entitlementsConfigProperties = entitlementsConfigProperties; + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java index 20236aaf..3103acc8 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java @@ -27,56 +27,61 @@ import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.concurrent.TimeoutException; import javax.annotation.PostConstruct; +import org.opengroup.osdu.indexer.config.RabbitMqConfigProperties; import org.opengroup.osdu.indexer.messagebus.IMessageFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class RabbitMQFactoryImpl implements IMessageFactory { - private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + private final RabbitMqConfigProperties rabbitMqConfigProperties; - @Value("${mb.rabbitmq.uri}") - private String uri; + private Channel channel; - private Channel channel; + @Autowired + public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) { + this.rabbitMqConfigProperties = rabbitMqConfigProperties; + } - @PostConstruct - private void init() { - ConnectionFactory factory = new ConnectionFactory(); - try { - LOG.debug("RabbitMQ Channel " + uri); - factory.setUri(uri); - factory.setAutomaticRecoveryEnabled(true); - Connection conn = factory.newConnection(); - this.channel = conn.createChannel(); - LOG.debug("RabbitMQ Channel was created."); - for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { - channel.queueDeclare(queue, true, false, false, null); - LOG.debug("Queue [" + queue + "] was declared."); - } - } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { - LOG.error(e.getMessage(), e); - } + @PostConstruct + private void init() { + ConnectionFactory factory = new ConnectionFactory(); + try { + String uri = rabbitMqConfigProperties.getMbRabbitMqUri(); + LOG.debug(String.format("RabbitMQ Channel = %s", uri)); + factory.setUri(uri); + factory.setAutomaticRecoveryEnabled(true); + Connection conn = factory.newConnection(); + this.channel = conn.createChannel(); + LOG.debug("RabbitMQ Channel was created."); + for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { + channel.queueDeclare(queue, true, false, false, null); + LOG.debug(String.format("Queue [%s] was declared.", queue)); + } + } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { + LOG.error(e.getMessage(), e); + } - } + } - @Override - public void sendMessage(String msg) { - this.sendMessage("records", msg); - } + @Override + public void sendMessage(String msg) { + this.sendMessage("records", msg); + } - @Override - public void sendMessage(String queueName, String msg) { - String queueNameWithPrefix = queueName; - try { - channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); - LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); - } catch (IOException e) { - LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]"); - LOG.error(e.getMessage(), e); - } - } + @Override + public void sendMessage(String queueName, String msg) { + String queueNameWithPrefix = queueName; + try { + channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); + LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix)); + } catch (IOException e) { + LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix)); + LOG.error(e.getMessage(), e); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java index 8aad1a1a..b60b6e59 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,58 +40,63 @@ import org.springframework.stereotype.Component; @Component public class TenantFactoryImpl implements ITenantFactory { - private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); - - public static final String MAIN_DATABASE = "main"; - public static final String TENANT_INFO = "tenantinfo"; - - @Autowired - private MongoDdmsClient mongoClient; - - private Map tenants; - - public boolean exists(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.containsKey(tenantName); - } - - public TenantInfo getTenantInfo(String tenantName) { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.get(tenantName); - } - - public Collection listTenantInfo() { - if (this.tenants == null) { - initTenants(); - } - return this.tenants.values(); - } - - public ICache createCache(String tenantName, String host, int port, - int expireTimeSeconds, Class classOfV) { - return null; - } - - public void flushCache() { - } - - private void initTenants() { - this.tenants = new HashMap<>(); - MongoCollection mongoCollection = mongoClient - .getMongoCollection(MAIN_DATABASE, TENANT_INFO); - FindIterable results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { - LOG.error(String.format("Collection \'%s\' is empty.", results)); - } - for (Document document : results) { - TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class); - this.tenants.put(tenantInfo.getName(), tenantInfo); - } - } + private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); + + public static final String MAIN_DATABASE = "main"; + public static final String TENANT_INFO = "tenantinfo"; + + + private final MongoDdmsClient mongoClient; + + @Autowired + public TenantFactoryImpl(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } + + private Map tenants; + + public boolean exists(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.containsKey(tenantName); + } + + public TenantInfo getTenantInfo(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.get(tenantName); + } + + public Collection listTenantInfo() { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.values(); + } + + public ICache createCache(String tenantName, String host, int port, + int expireTimeSeconds, Class classOfV) { + return null; + } + + public void flushCache() { + } + + private void initTenants() { + this.tenants = new HashMap<>(); + MongoCollection mongoCollection = mongoClient + .getMongoCollection(MAIN_DATABASE, TENANT_INFO); + FindIterable results = mongoCollection.find(); + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + } + for (Document document : results) { + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class); + this.tenants.put(tenantInfo.getName(), tenantInfo); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java deleted file mode 100644 index 2adc6336..00000000 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.kms; - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.cloudkms.v1.CloudKMS; -import com.google.api.services.cloudkms.v1.CloudKMSScopes; -import com.google.api.services.cloudkms.v1.model.DecryptRequest; -import com.google.api.services.cloudkms.v1.model.DecryptResponse; -import com.google.api.services.cloudkms.v1.model.EncryptRequest; -import com.google.api.services.cloudkms.v1.model.EncryptResponse; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; -import org.opengroup.osdu.core.common.search.Preconditions; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -@Component -@RequestScope -public class KmsClient implements IKmsClient { - - @Value("${GOOGLE_CLOUD_PROJECT}") - private String GOOGLE_CLOUD_PROJECT; - - @Value("${KMS_KEY}") - private String KMS_KEY; - - @Value("${KEY_RING}") - private String KEY_RING; - - private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s"; - - /** - * Encrypts the given plaintext using the specified crypto key. - * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation - */ - public String encryptString(String textToBeEncrypted) throws IOException { - Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null"); - - byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8); - String resourceName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); - CloudKMS kms = createAuthorizedClient(); - EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext); - EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys() - .encrypt(resourceName, request) - .execute(); - return response.getCiphertext(); - } - - /** - * Decrypts the provided ciphertext with the specified crypto key. - * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled - */ - public String decryptString(String textToBeDecrypted) throws IOException { - Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null"); - - CloudKMS kms = createAuthorizedClient(); - String cryptoKeyName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY); - DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted); - DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys() - .decrypt(cryptoKeyName, request) - .execute(); - return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim(); - } - - /** - * Creates an authorized CloudKMS client service using Application Default Credentials. - * - * @return an authorized CloudKMS client - * @throws IOException if there's an error getting the default credentials. - */ - private CloudKMS createAuthorizedClient() throws IOException { - HttpTransport transport = new NetHttpTransport(); - JsonFactory jsonFactory = new JacksonFactory(); - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - credential = credential.createScoped(CloudKMSScopes.all()); - } - return new CloudKMS.Builder(transport, jsonFactory, credential) - .setApplicationName("CloudKMS snippets") - .build(); - } -} \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java index 1679a6d6..21316cc6 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java index 95bbd9be..12cffcf0 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,104 +18,110 @@ package org.opengroup.osdu.indexer.middleware; import com.google.common.base.Strings; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.extern.java.Log; import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.http.ResponseHeaders; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; -import javax.inject.Inject; -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - @Log @Component public class IndexFilter implements Filter { - @Inject - private DpsHeaders dpsHeaders; - - @Inject - private IRequestInfo requestInfo; - @Value("${DEPLOYMENT_ENVIRONMENT}") - private String DEPLOYMENT_ENVIRONMENT; - - private FilterConfig filterConfig; - - private static final String PATH_SWAGGER = "/swagger.json"; - private static final String PATH_TASK_HANDLERS = "task-handlers"; - private static final String PATH_CRON_HANDLERS = "cron-handlers"; + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final DpsHeaders dpsHeaders; + private final IRequestInfo requestInfo; + + private FilterConfig filterConfig; + + private static final String PATH_SWAGGER = "/swagger.json"; + private static final String PATH_TASK_HANDLERS = "task-handlers"; + private static final String PATH_CRON_HANDLERS = "cron-handlers"; + + @Autowired + public IndexFilter(IndexerConfigurationProperties indexerConfigurationProperties, + DpsHeaders dpsHeaders, IRequestInfo requestInfo) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.dpsHeaders = dpsHeaders; + this.requestInfo = requestInfo; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, + FilterChain filterChain) + throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + String uri = httpRequest.getRequestURI().toLowerCase(); + + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri + .contains(PATH_TASK_HANDLERS)) { + if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment()) + != DeploymentEnvironment.LOCAL) { + checkWorkerApiAccess(requestInfo); + } + } - @Override - public void init(FilterConfig filterConfig) throws ServletException { + if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri + .contains(PATH_CRON_HANDLERS)) { + checkWorkerApiAccess(requestInfo); } - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; - String uri = httpRequest.getRequestURI().toLowerCase(); - - if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) { - if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) != DeploymentEnvironment.LOCAL) { - checkWorkerApiAccess(requestInfo); - } - } - - if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) { - checkWorkerApiAccess(requestInfo); - } - -// if (!httpRequest.isSecure()) { -// throw new AppException(302, "Redirect", "HTTP is not supported. Use HTTPS."); -// } - - filterChain.doFilter(servletRequest, servletResponse); - - HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; - Map> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; - for (Map.Entry> header : standardHeaders.entrySet()) { - httpResponse.addHeader(header.getKey(), header.getValue().toString()); - } - if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) { - httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId()); - } + filterChain.doFilter(servletRequest, servletResponse); + HttpServletResponse httpResponse = (HttpServletResponse) servletResponse; + Map> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; + for (Map.Entry> header : standardHeaders.entrySet()) { + httpResponse.addHeader(header.getKey(), header.getValue().toString()); } - - @Override - public void destroy() { + if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) { + httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId()); } + } - private void checkWorkerApiAccess(IRequestInfo requestInfo) { -// if (requestInfo.isTaskQueueRequest()) return; -// throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only"); - } + @Override + public void destroy() { + } - private List validateAccountId(DpsHeaders requestHeaders) { - String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId(); - String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader); + private void checkWorkerApiAccess(IRequestInfo requestInfo) { + } - if (Strings.isNullOrEmpty(accountHeader)) { - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); - } + private List validateAccountId(DpsHeaders requestHeaders) { + String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId(); + String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader); - List dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*")); - if (dataPartitions.isEmpty() || dataPartitions.size() > 1) { - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo); - } - return dataPartitions; + if (Strings.isNullOrEmpty(accountHeader)) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", + "invalid or empty data partition", debuggingInfo); } + List dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*")); + if (dataPartitions.isEmpty() || dataPartitions.size() > 1) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", + "invalid or empty data partition", debuggingInfo); + } + return dataPartitions; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java index c3285bd7..9b7f8b1f 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,82 +38,86 @@ import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; public class DatastoreCredential extends GoogleCredentials { - private static final long serialVersionUID = 8344377091688956815L; - private static final JsonFactory JSON_FACTORY = new JacksonFactory(); - private Iam iam; + private static final long serialVersionUID = 8344377091688956815L; + private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + private Iam iam; - private final TenantInfo tenant; - private final DatastoreCredentialCache cache; + private final TenantInfo tenant; + private final DatastoreCredentialCache cache; - protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { - this.tenant = tenant; - this.cache = cache; - } + protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) { + this.tenant = tenant; + this.cache = cache; + } - @Override - public AccessToken refreshAccessToken() { + @Override + public AccessToken refreshAccessToken() { - String cacheKey = this.getCacheKey(); + String cacheKey = this.getCacheKey(); - AccessToken accessToken = this.cache.get(cacheKey); + AccessToken accessToken = this.cache.get(cacheKey); - if (accessToken != null) { - return accessToken; - } + if (accessToken != null) { + return accessToken; + } - try { - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(this.getPayload()); + try { + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(this.getPayload()); - String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); + String serviceAccountName = String + .format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount()); - SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest); + SignJwt signJwt = this.getIam().projects().serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); - accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); + accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600)); - this.cache.put(cacheKey, accessToken); + this.cache.put(cacheKey, accessToken); - return accessToken; - } catch (Exception e) { - throw new RuntimeException("Error creating datastore credential", e); - } - } + return accessToken; + } catch (Exception e) { + throw new RuntimeException("Error creating datastore credential", e); + } + } - private String getPayload() { - JsonObject payload = new JsonObject(); - payload.addProperty("iss", this.tenant.getServiceAccount()); - payload.addProperty("sub", this.tenant.getServiceAccount()); - payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); - payload.addProperty("iat", System.currentTimeMillis() / 1000); + private String getPayload() { + JsonObject payload = new JsonObject(); + payload.addProperty("iss", this.tenant.getServiceAccount()); + payload.addProperty("sub", this.tenant.getServiceAccount()); + payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore"); + payload.addProperty("iat", System.currentTimeMillis() / 1000); - return payload.toString(); - } + return payload.toString(); + } - protected void setIam(Iam iam) { - this.iam = iam; - } + protected void setIam(Iam iam) { + this.iam = iam; + } - private Iam getIam() throws Exception { - if (this.iam == null) { + private Iam getIam() throws Exception { + if (this.iam == null) { - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); - } + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM)); + } - Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY, - credential).setApplicationName("Search Service"); + Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), + JSON_FACTORY, + credential).setApplicationName("Search Service"); - this.iam = builder.build(); - } - return this.iam; - } + this.iam = builder.build(); + } + return this.iam; + } - private String getCacheKey() { - return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); - } + private String getCacheKey() { + return Crc32c + .hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName())); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java index db90ee7e..144edee0 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,49 +22,54 @@ import com.google.cloud.TransportOptions; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.http.HttpTransportOptions; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.threeten.bp.Duration; -import javax.inject.Inject; -import java.util.HashMap; -import java.util.Map; - @Component public class DatastoreFactory { - @Inject - private DatastoreCredentialCache cache; + @Inject + private DatastoreCredentialCache cache; + + @Autowired + public DatastoreFactory(DatastoreCredentialCache cache) { + this.cache = cache; + } - private static Map DATASTORE_CLIENTS = new HashMap<>(); + private static Map DATASTORE_CLIENTS = new HashMap<>(); - private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofSeconds(10)) - .setMaxRetryDelay(Duration.ofSeconds(32)) - .setRetryDelayMultiplier(2.0) - .setTotalTimeout(Duration.ofSeconds(50)) - .setInitialRpcTimeout(Duration.ofSeconds(50)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(50)) - .build(); + private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() + .setMaxAttempts(6) + .setInitialRetryDelay(Duration.ofSeconds(10)) + .setMaxRetryDelay(Duration.ofSeconds(32)) + .setRetryDelayMultiplier(2.0) + .setTotalTimeout(Duration.ofSeconds(50)) + .setInitialRpcTimeout(Duration.ofSeconds(50)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeout(Duration.ofSeconds(50)) + .build(); - private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() - .setReadTimeout(30000) - .build(); + private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder() + .setReadTimeout(30000) + .build(); - public Datastore getDatastoreInstance(TenantInfo tenantInfo) { - if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { - Datastore googleDatastore = DatastoreOptions.newBuilder() - .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) - .setRetrySettings(RETRY_SETTINGS) - .setTransportOptions(TRANSPORT_OPTIONS) - .setNamespace(tenantInfo.getName()) - .setProjectId(tenantInfo.getProjectId()) - .build().getService(); - DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); - } - return DATASTORE_CLIENTS.get(tenantInfo.getName()); + public Datastore getDatastoreInstance(TenantInfo tenantInfo) { + if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) { + Datastore googleDatastore = DatastoreOptions.newBuilder() + .setCredentials(new DatastoreCredential(tenantInfo, this.cache)) + .setRetrySettings(RETRY_SETTINGS) + .setTransportOptions(TRANSPORT_OPTIONS) + .setNamespace(tenantInfo.getName()) + .setProjectId(tenantInfo.getProjectId()) + .build().getService(); + DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore); } + return DATASTORE_CLIENTS.get(tenantInfo.getName()); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java index a3ab89c5..1c4d4fba 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package org.opengroup.osdu.indexer.persistence; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import java.util.Objects; @@ -27,7 +26,6 @@ import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -36,53 +34,54 @@ import org.springframework.stereotype.Component; @Component public class ElasticRepositoryMongoDB implements IElasticRepository { - private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class); + private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class); - private static final String SCHEMA_DATABASE = "local"; - private static final String SEARCH_SETTINGS = "SearchSettings"; - private static final String HOST = "host"; - private static final String PORT = "port"; - private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; + private static final String SCHEMA_DATABASE = "local"; + private static final String SEARCH_SETTINGS = "SearchSettings"; + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration"; - @Autowired - private MongoDdmsClient mongoClient; + private final MongoDdmsClient mongoClient; - @Autowired - private IKmsClient kmsClient; + @Autowired + public ElasticRepositoryMongoDB(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } - @Override - public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { - MongoCollection mongoCollection = mongoClient - .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS); + @Override + public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) { + MongoCollection mongoCollection = mongoClient + .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS); - FindIterable results = mongoCollection.find(); + FindIterable results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { - LOG.error(String.format("Collection \'%s\' is empty.", results)); - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", - "An error has occurred fetching cluster settings from the database."); - } + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database."); + } - Document document = results.first(); + Document document = results.first(); - String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString(); - String encryptedHost = document.get(HOST).toString(); - String encryptedPort = document.get(PORT).toString(); + String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString(); + String encryptedHost = document.get(HOST).toString(); + String encryptedPort = document.get(PORT).toString(); - try { + try { - String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost); - String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort); - String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration); + String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost); + String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort); + String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration); - int port = Integer.parseInt(portString); - ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword); - clusterSettings.setHttps(false); - return clusterSettings; + int port = Integer.parseInt(portString); + ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword); + clusterSettings.setHttps(false); + return clusterSettings; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", - "An error has occurred fetching cluster settings from the database.", e); - } - } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", + "An error has occurred fetching cluster settings from the database.", e); + } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java index 43b3fd5b..6cb02a3a 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,14 +27,15 @@ import org.springframework.stereotype.Component; @Component public class MongoDdmsClient { - @Autowired - private MongoClientHandler mongoClientHandler; + private final MongoClientHandler mongoClientHandler; @Autowired - private TenantInfo tenantInfo; + public MongoDdmsClient(MongoClientHandler mongoClientHandler) { + this.mongoClientHandler = mongoClientHandler; + } - public MongoCollection getMongoCollection(String dbName, String collectionName) { - return mongoClientHandler.getMongoClient().getDatabase(dbName) - .getCollection(collectionName); - } + public MongoCollection getMongoCollection(String dbName, String collectionName) { + return mongoClientHandler.getMongoClient().getDatabase(dbName) + .getCollection(collectionName); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java index d2a4d74c..a9a82d8c 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,61 +43,67 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope public class PublisherImpl implements IPublisher { - private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class); - - @Autowired - IMessageFactory mq; - - - @Override - public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { - - String tenant = headers.getPartitionId(); - if (Strings.isNullOrEmpty(tenant)) { - tenant = headers.getAccountId(); - } - - Map message = new HashMap<>(); - message.put(tenant, headers.getPartitionIdWithFallbackToAccountId()); - headers.addCorrelationIdIfMissing(); - message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - - RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus); - message.put("data", recordChangedMessages.toString()); - - try { - LOG.info("Indexer publishes message " + headers.getCorrelationId()); - mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString()); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - - private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) { - - Gson gson = new GsonBuilder().create(); - Map attributesMap = new HashMap<>(); - Type listType = new TypeToken>() { - }.getType(); - - JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType); - String statusChangedTagsData = (statusChangedTagsJson.toString()); - - String tenant = headers.getPartitionId(); - // This code it to provide backward compatibility to slb-account-id - if (!Strings.isNullOrEmpty(tenant)) { - attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); - } else { - attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); - } - headers.addCorrelationIdIfMissing(); - attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - - RecordChangedMessages recordChangedMessages = new RecordChangedMessages(); - // statusChangedTagsData is not ByteString but String - recordChangedMessages.setData(statusChangedTagsData); - recordChangedMessages.setAttributes(attributesMap); - - return recordChangedMessages; - } + private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class); + private final IMessageFactory mq; + + @Autowired + public PublisherImpl(IMessageFactory mq) { + this.mq = mq; + } + + @Override + public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) { + + String tenant = headers.getPartitionId(); + if (Strings.isNullOrEmpty(tenant)) { + tenant = headers.getAccountId(); + } + + Map message = new HashMap<>(); + message.put(tenant, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, + indexerBatchStatus); + message.put("data", recordChangedMessages.toString()); + + try { + LOG.info("Indexer publishes message " + headers.getCorrelationId()); + mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, + JobStatus indexerBatchStatus) { + + Gson gson = new GsonBuilder().create(); + Map attributesMap = new HashMap<>(); + Type listType = new TypeToken>() { + }.getType(); + + JsonElement statusChangedTagsJson = gson + .toJsonTree(indexerBatchStatus.getStatusesList(), listType); + String statusChangedTagsData = (statusChangedTagsJson.toString()); + + String tenant = headers.getPartitionId(); + // This code it to provide backward compatibility to slb-account-id + if (!Strings.isNullOrEmpty(tenant)) { + attributesMap + .put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + } else { + attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + } + headers.addCorrelationIdIfMissing(); + attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + RecordChangedMessages recordChangedMessages = new RecordChangedMessages(); + // statusChangedTagsData is not ByteString but String + recordChangedMessages.setData(statusChangedTagsData); + recordChangedMessages.setAttributes(attributesMap); + + return recordChangedMessages; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java index 7927d6f3..0db2f9e5 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,18 +29,16 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur @EnableGlobalMethodSecurity(prePostEnabled = true) public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { - @Override - protected void configure(HttpSecurity http) throws Exception { - - http.httpBasic().disable() - .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy - - } - - @Override - public void configure(WebSecurity web) throws Exception { - web.ignoring().antMatchers("/api-docs") - .antMatchers("/swagger"); - } + @Override + protected void configure(HttpSecurity http) throws Exception { + http.httpBasic().disable() + .csrf().disable(); + } + + @Override + public void configure(WebSecurity web) throws Exception { + web.ignoring().antMatchers("/api-docs") + .antMatchers("/swagger"); + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 0b558b08..558e30bc 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -1,16 +1,19 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.opengroup.osdu.indexer.service; @@ -68,458 +71,544 @@ import org.springframework.stereotype.Service; @Primary public class IndexerServiceImpl implements IndexerService { - private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); - - private static final List RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE)); - - private final Gson gson = new Gson(); - - @Inject - private JaxRsDpsLog jaxRsDpsLog; - @Inject - private AuditLogger auditLogger; - @Inject - private StorageService storageService; - @Inject - private IndexSchemaService schemaService; - @Inject - private IndicesService indicesService; - @Inject - private IndexerMappingService mappingService; - @Inject - private IPublisher progressPublisher; - @Inject - private ElasticClientHandler elasticClientHandler; - @Inject - private IndexerQueueTaskBuilder indexerQueueTaskBuilder; - @Inject - private ElasticIndexNameResolver elasticIndexNameResolver; - @Inject - private StorageIndexerPayloadMapper storageIndexerPayloadMapper; - @Inject - private IRequestInfo requestInfo; - @Inject - private JobStatus jobStatus; - - private DpsHeaders headers; - - @Override - public JobStatus processRecordChangedMessages(RecordChangedMessages message, List recordInfos) throws Exception { - - // this should not happen - if (recordInfos.size() == 0) return null; - - String errorMessage = ""; - List retryRecordIds = new LinkedList<>(); - - // get auth header with service account Authorization - this.headers = this.requestInfo.getHeaders(); - String authorization = message.getAttributes().get("authorization"); - headers.put("authorization", authorization); - // initialize status for all messages. - this.jobStatus.initialize(recordInfos); - - try { - // get upsert records - Map> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos); - if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) { - List upsertFailureRecordIds = processUpsertRecords(upsertRecordMap); - retryRecordIds.addAll(upsertFailureRecordIds); - } - - // get delete records - Map> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos); - if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) { - List deleteFailureRecordIds = processDeleteRecords(deleteRecordMap); - retryRecordIds.addAll(deleteFailureRecordIds); - } - - // process schema change messages - Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); - if (schemaMsgs != null && !schemaMsgs.isEmpty()) { - this.schemaService.processSchemaMessages(schemaMsgs); - } - - // process failed records - if (retryRecordIds.size() > 0) { - retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message); - } - } catch (IOException e) { - errorMessage = e.getMessage(); - throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e); - } catch (AppException e) { - errorMessage = e.getMessage(); - throw e; - } catch (Exception e) { - errorMessage = "error indexing records"; - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e); - } finally { - this.jobStatus.finalizeRecordStatus(errorMessage); - this.updateAuditLog(); - this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus); - } - - return jobStatus; + private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1); + + private static final List RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays + .asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, + RestStatus.SERVICE_UNAVAILABLE)); + + private final Gson gson = new Gson(); + + @Inject + private JaxRsDpsLog jaxRsDpsLog; + @Inject + private AuditLogger auditLogger; + @Inject + private StorageService storageService; + @Inject + private IndexSchemaService schemaService; + @Inject + private IndicesService indicesService; + @Inject + private IndexerMappingService mappingService; + @Inject + private IPublisher progressPublisher; + @Inject + private ElasticClientHandler elasticClientHandler; + @Inject + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Inject + private ElasticIndexNameResolver elasticIndexNameResolver; + @Inject + private StorageIndexerPayloadMapper storageIndexerPayloadMapper; + @Inject + private IRequestInfo requestInfo; + @Inject + private JobStatus jobStatus; + + private DpsHeaders headers; + + @Override + public JobStatus processRecordChangedMessages(RecordChangedMessages message, + List recordInfos) throws Exception { + + // this should not happen + if (recordInfos.size() == 0) { + return null; } - @Override - public void processSchemaMessages(List recordInfos) throws IOException { - Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); - if (schemaMsgs != null && !schemaMsgs.isEmpty()) { - try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { - schemaMsgs.entrySet().forEach(msg -> { - try { - processSchemaEvents(restClient, msg); - } catch (IOException | ElasticsearchStatusException e) { - throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage()); - } - }); - } - } + String errorMessage = ""; + List retryRecordIds = new LinkedList<>(); + + // get auth header with service account Authorization + this.headers = this.requestInfo.getHeaders(); + String authorization = message.getAttributes().get("authorization"); + headers.put("authorization", authorization); + // initialize status for all messages. + this.jobStatus.initialize(recordInfos); + + try { + // get upsert records + Map> upsertRecordMap = RecordInfo + .getUpsertRecordIds(recordInfos); + if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) { + List upsertFailureRecordIds = processUpsertRecords(upsertRecordMap); + retryRecordIds.addAll(upsertFailureRecordIds); + } + + // get delete records + Map> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos); + if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) { + List deleteFailureRecordIds = processDeleteRecords(deleteRecordMap); + retryRecordIds.addAll(deleteFailureRecordIds); + } + + // process schema change messages + Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + this.schemaService.processSchemaMessages(schemaMsgs); + } + + // process failed records + if (retryRecordIds.size() > 0) { + retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message); + } + } catch (IOException e) { + errorMessage = e.getMessage(); + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", + errorMessage, e); + } catch (AppException e) { + errorMessage = e.getMessage(); + throw e; + } catch (Exception e) { + errorMessage = "error indexing records"; + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", + "An unknown error has occurred.", e); + } finally { + this.jobStatus.finalizeRecordStatus(errorMessage); + this.updateAuditLog(); + this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus); } - private void processSchemaEvents(RestHighLevelClient restClient, - Map.Entry msg) throws IOException, ElasticsearchStatusException { - String kind = msg.getKey(); - String index = elasticIndexNameResolver.getIndexNameFromKind(kind); - - boolean indexExist = indicesService.isIndexExist(restClient, index); - if (indexExist && msg.getValue() == OperationType.purge_schema) { - indicesService.deleteIndex(restClient, index); - } + return jobStatus; + } + + @Override + public void processSchemaMessages(List recordInfos) throws IOException { + Map schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { + schemaMsgs.entrySet().forEach(msg -> { + try { + processSchemaEvents(restClient, msg); + } catch (IOException | ElasticsearchStatusException e) { + throw new AppException( + org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), + "unable to process schema delete", e.getMessage()); + } + }); + } } + } - private List processUpsertRecords(Map> upsertRecordMap) throws Exception { - // get schema for kind - Map schemas = this.getSchema(upsertRecordMap); - - if (schemas.isEmpty()) return new LinkedList<>(); - - // get recordIds with valid upsert index-status - List recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus(); - - if (recordIds.isEmpty()) return new LinkedList<>(); - - // get records via storage api - Records storageRecords = this.storageService.getStorageRecords(recordIds); - List failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords()); - - // map storage records to indexer payload - RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords); + private void processSchemaEvents(RestHighLevelClient restClient, + Map.Entry msg) throws IOException, ElasticsearchStatusException { + String kind = msg.getKey(); + String index = elasticIndexNameResolver.getIndexNameFromKind(kind); - jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size())); + boolean indexExist = indicesService.isIndexExist(restClient, index); + if (indexExist && msg.getValue() == OperationType.purge_schema) { + indicesService.deleteIndex(restClient, index); + } + } - // index records - failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload)); + private List processUpsertRecords(Map> upsertRecordMap) + throws Exception { + // get schema for kind + Map schemas = this.getSchema(upsertRecordMap); - return failedOrRetryRecordIds; + if (schemas.isEmpty()) { + return new LinkedList<>(); } - private Map getSchema(Map> upsertRecordMap) { + // get recordIds with valid upsert index-status + List recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus(); - Map schemas = new HashMap<>(); + if (recordIds.isEmpty()) { + return new LinkedList<>(); + } - try { - for (Map.Entry> entry : upsertRecordMap.entrySet()) { + // get records via storage api + Records storageRecords = this.storageService.getStorageRecords(recordIds); + List failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords()); - String kind = entry.getKey(); - IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); - if (schemaObj.isDataSchemaMissing()) { - this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind)); - } + // map storage records to indexer payload + RecordIndexerPayload recordIndexerPayload = this + .getIndexerPayload(upsertRecordMap, schemas, storageRecords); - schemas.put(kind, schemaObj); - } - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e); - } - - return schemas; - } + jaxRsDpsLog.info(String.format( + "records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", + recordIds.size(), storageRecords.getRecords().size(), + recordIndexerPayload.getRecords().size())); - private RecordIndexerPayload getIndexerPayload(Map> upsertRecordMap, Map kindSchemaMap, Records records) { - List storageValidRecords = records.getRecords(); - List indexerPayload = new ArrayList<>(); - List schemas = new ArrayList<>(); + // index records + failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload)); - for (Records.Entity storageRecord : storageValidRecords) { + return failedOrRetryRecordIds; + } - Map idOperationMap = upsertRecordMap.get(storageRecord.getKind()); + private Map getSchema( + Map> upsertRecordMap) { - // skip if storage returned record with same id but different kind - if (idOperationMap == null) { - String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind()); - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId())); - continue; - } + Map schemas = new HashMap<>(); - IndexSchema schema = kindSchemaMap.get(storageRecord.getKind()); - schemas.add(schema); + try { + for (Map.Entry> entry : upsertRecordMap.entrySet()) { - // skip indexing of records if data block is empty - RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap); - if (document != null) { - indexerPayload.add(document); - } + String kind = entry.getKey(); + IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false); + if (schemaObj.isDataSchemaMissing()) { + this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, "schema not found", + String.format("schema not found | kind: %s", kind)); } - // this should only happen if storage service returned WRONG records with kind for all the records in the messages - if (indexerPayload.isEmpty()) { - throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records"); - } + schemas.put(kind, schemaObj); + } + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", + "An error has occurred while getting schema", e); + } - return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build(); + return schemas; + } + + private RecordIndexerPayload getIndexerPayload( + Map> upsertRecordMap, + Map kindSchemaMap, Records records) { + List storageValidRecords = records.getRecords(); + List indexerPayload = new ArrayList<>(); + List schemas = new ArrayList<>(); + + for (Records.Entity storageRecord : storageValidRecords) { + + Map idOperationMap = upsertRecordMap.get(storageRecord.getKind()); + + // skip if storage returned record with same id but different kind + if (idOperationMap == null) { + String message = String.format( + "storage service returned incorrect record | requested kind: %s | received kind: %s", + this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind()); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, + RequestStatus.STORAGE_CONFLICT, message, + String.format("%s | record-id: %s", message, storageRecord.getId())); + continue; + } + + IndexSchema schema = kindSchemaMap.get(storageRecord.getKind()); + schemas.add(schema); + + // skip indexing of records if data block is empty + RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, + idOperationMap); + if (document != null) { + indexerPayload.add(document); + } } - private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map idToOperationMap) { - - RecordIndexerPayload.Record document = null; - - try { - Map storageRecordData = storageRecord.getData(); - document = new RecordIndexerPayload.Record(); - if (storageRecordData == null || storageRecordData.isEmpty()) { - String message = "empty or null data block found in the storage record"; - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); - } else if (schemaObj.isDataSchemaMissing()) { - document.setSchemaMissing(true); - } else { - Map dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId()); - if (dataMap.isEmpty()) { - document.setMappingMismatch(true); - String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData); - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message)); - } - document.setData(dataMap); - } - } catch (AppException e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); - jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e); - } catch (Exception e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage())); - jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e); - } + // this should only happen if storage service returned WRONG records with kind for all the records in the messages + if (indexerPayload.isEmpty()) { + throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", + "upsert record failed, storage service returned incorrect records"); + } - try { - // index individual parts of kind - String[] kindParts = storageRecord.getKind().split(":"); - - document.setKind(storageRecord.getKind()); - document.setNamespace(kindParts[0] + ":" + kindParts[1]); - document.setType(kindParts[2]); - document.setId(storageRecord.getId()); - document.setVersion(storageRecord.getVersion()); - document.setAcl(storageRecord.getAcl()); - document.setLegal(storageRecord.getLegal()); - RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); - if (recordStatus.getIndexProgress().getStatusCode() == 0) { - recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); - } - document.setIndexProgress(recordStatus.getIndexProgress()); - if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry()); - document.setOperationType(idToOperationMap.get(storageRecord.getId())); - } catch (Exception e) { - this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage())); - jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e); + return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build(); + } + + private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, + Records.Entity storageRecord, Map idToOperationMap) { + + RecordIndexerPayload.Record document = null; + + try { + Map storageRecordData = storageRecord.getData(); + document = new RecordIndexerPayload.Record(); + if (storageRecordData == null || storageRecordData.isEmpty()) { + String message = "empty or null data block found in the storage record"; + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, message, + String.format("record-id: %s | %s", storageRecord.getId(), message)); + } else if (schemaObj.isDataSchemaMissing()) { + document.setSchemaMissing(true); + } else { + Map dataMap = this.storageIndexerPayloadMapper + .mapDataPayload(schemaObj, storageRecordData, storageRecord.getId()); + if (dataMap.isEmpty()) { + document.setMappingMismatch(true); + String message = String.format( + "complete schema mismatch: none of the data attribute can be mapped | data: %s", + storageRecordData); + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, + HttpStatus.SC_NOT_FOUND, message, + String.format("record-id: %s | %s", storageRecord.getId(), message)); } - return document; + document.setData(dataMap); + } + } catch (AppException e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + jaxRsDpsLog + .warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, + String.format("error parsing records against schema, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String + .format("record-id: %s | error parsing records against schema, error-message: %s", + storageRecord.getId(), e.getMessage()), e); } - private List processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception { - - try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { - List schemas = recordIndexerPayload.getSchemas(); - if (schemas == null || schemas.isEmpty()) { - return new LinkedList<>(); - } - - // process the schema - this.cacheOrCreateElasticMapping(schemas, restClient); - - // process the records - return this.upsertRecords(recordIndexerPayload.getRecords(), restClient); - } + try { + // index individual parts of kind + String[] kindParts = storageRecord.getKind().split(":"); + + document.setKind(storageRecord.getKind()); + document.setNamespace(kindParts[0] + ":" + kindParts[1]); + document.setType(kindParts[2]); + document.setId(storageRecord.getId()); + document.setVersion(storageRecord.getVersion()); + document.setAcl(storageRecord.getAcl()); + document.setLegal(storageRecord.getLegal()); + RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId()); + if (recordStatus.getIndexProgress().getStatusCode() == 0) { + recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK); + } + document.setIndexProgress(recordStatus.getIndexProgress()); + if (storageRecord.getAncestry() != null) { + document.setAncestry(storageRecord.getAncestry()); + } + document.setOperationType(idToOperationMap.get(storageRecord.getId())); + } catch (Exception e) { + this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, + HttpStatus.SC_INTERNAL_SERVER_ERROR, + String.format("error parsing meta data, error-message: %s", e.getMessage())); + jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", + storageRecord.getId(), e.getMessage()), e); } + return document; + } - private void cacheOrCreateElasticMapping(List schemas, RestHighLevelClient restClient) throws Exception { + private List processElasticMappingAndUpsertRecords( + RecordIndexerPayload recordIndexerPayload) throws Exception { - for (IndexSchema schema : schemas) { - String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind()); + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + List schemas = recordIndexerPayload.getSchemas(); + if (schemas == null || schemas.isEmpty()) { + return new LinkedList<>(); + } - // check if index exist - if (this.indicesService.isIndexExist(restClient, index)) continue; + // process the schema + this.cacheOrCreateElasticMapping(schemas, restClient); - // create index - Map mapping = this.mappingService.getIndexMappingFromRecordSchema(schema); - if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index)); - } - } + // process the records + return this.upsertRecords(recordIndexerPayload.getRecords(), restClient); + } + } + + private void cacheOrCreateElasticMapping(List schemas, + RestHighLevelClient restClient) throws Exception { + + for (IndexSchema schema : schemas) { + String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind()); + + // check if index exist + if (this.indicesService.isIndexExist(restClient, index)) { + continue; + } + + // create index + Map mapping = this.mappingService.getIndexMappingFromRecordSchema(schema); + if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", + "Error creating index.", + String.format("Failed to get confirmation from elastic server for index: %s", index)); + } } + } - private List upsertRecords(List records, RestHighLevelClient restClient) throws AppException { - if (records == null || records.isEmpty()) return new LinkedList<>(); - - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.timeout(BULK_REQUEST_TIMEOUT); - - for (RecordIndexerPayload.Record record : records) { - if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) { - // it will come here when schema is missing - // TODO: rollback once we know what is causing the problem - jaxRsDpsLog.warning(String.format("data not found for record: %s", record)); - } - - OperationType operation = record.getOperationType(); - Map sourceMap = getSourceMap(record); - String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); - - if (operation == OperationType.create) { - IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON); - bulkRequest.add(indexRequest); - } else if (operation == OperationType.update) { - UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON); - bulkRequest.add(updateRequest); - } - } + private List upsertRecords(List records, + RestHighLevelClient restClient) throws AppException { + if (records == null || records.isEmpty()) { + return new LinkedList<>(); + } - return processBulkRequest(restClient, bulkRequest); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + + for (RecordIndexerPayload.Record record : records) { + if ((record.getData() == null || record.getData().isEmpty()) && !record + .skippedDataIndexing()) { + // it will come here when schema is missing + // TODO: rollback once we know what is causing the problem + jaxRsDpsLog.warning(String.format("data not found for record: %s", record)); + } + + OperationType operation = record.getOperationType(); + Map sourceMap = getSourceMap(record); + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind()); + + if (operation == OperationType.create) { + IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()) + .source(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(indexRequest); + } else if (operation == OperationType.update) { + UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()) + .upsert(this.gson.toJson(sourceMap), XContentType.JSON); + bulkRequest.add(updateRequest); + } } - private List processDeleteRecords(Map> deleteRecordMap) throws Exception { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.timeout(BULK_REQUEST_TIMEOUT); + return processBulkRequest(restClient, bulkRequest); + } - for (Map.Entry> record : deleteRecordMap.entrySet()) { + private List processDeleteRecords(Map> deleteRecordMap) + throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.timeout(BULK_REQUEST_TIMEOUT); - String[] kindParts = record.getKey().split(":"); - String type = kindParts[2]; + for (Map.Entry> record : deleteRecordMap.entrySet()) { - String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); + String[] kindParts = record.getKey().split(":"); + String type = kindParts[2]; - for (String id : record.getValue()) { - DeleteRequest deleteRequest = new DeleteRequest(index, type, id); - bulkRequest.add(deleteRequest); - } - } + String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey()); - try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { - return processBulkRequest(restClient, bulkRequest); - } + for (String id : record.getValue()) { + DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + bulkRequest.add(deleteRequest); + } } - private List processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException { - - List failureRecordIds = new LinkedList<>(); - if (bulkRequest.numberOfActions() == 0) return failureRecordIds; - - - - try { - BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); - - // log failed bulk requests - ArrayList bulkFailures = new ArrayList<>(); - int succeededResponses = 0; - int failedResponses = 0; - - for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { - if (bulkItemResponse.isFailed()) { - BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); - bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage())); - this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage()); - if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) { - failureRecordIds.add(bulkItemResponse.getId()); - } - failedResponses++; - } else { - succeededResponses++; - this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully"); - } - } - if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures); - - jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses)); - } catch (IOException e) { - // throw explicit 504 for IOException - throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e); - } catch (ElasticsearchStatusException e) { - throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e); - } - return failureRecordIds; + try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) { + return processBulkRequest(restClient, bulkRequest); } + } - private Map getSourceMap(RecordIndexerPayload.Record record) { + private List processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) + throws AppException { - Map indexerPayload = new HashMap<>(); - - // get the key and get the corresponding object from the individualRecord object - if (record.getData() != null) { - Map data = new HashMap<>(); - for (Map.Entry entry : record.getData().entrySet()) { - data.put(entry.getKey(), entry.getValue()); - } - indexerPayload.put(Constants.DATA, data); - } + List failureRecordIds = new LinkedList<>(); + if (bulkRequest.numberOfActions() == 0) { + return failureRecordIds; + } - indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId()); - indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind()); - indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace()); - indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); - indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); - indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); - indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); - indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); - indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); - if (record.getAncestry() != null) { - indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry()); + try { + BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT); + + // log failed bulk requests + ArrayList bulkFailures = new ArrayList<>(); + int succeededResponses = 0; + int failedResponses = 0; + + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", + failure.getStatus(), failure.getId(), failure.getMessage())); + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, + failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage()); + if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) { + failureRecordIds.add(bulkItemResponse.getId()); + } + failedResponses++; + } else { + succeededResponses++; + this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, + HttpStatus.SC_OK, "Indexed Successfully"); } - return indexerPayload; + } + if (!bulkFailures.isEmpty()) { + this.jaxRsDpsLog.warning(bulkFailures); + } + + jaxRsDpsLog.info(String + .format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", + bulkRequest.numberOfActions(), succeededResponses, failedResponses)); + } catch (IOException e) { + // throw explicit 504 for IOException + throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", + "Request cannot be completed in specified time.", e); + } catch (ElasticsearchStatusException e) { + throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", + "Error indexing records.", e); } + return failureRecordIds; + } - private void retryAndEnqueueFailedRecords(List recordInfos, List failuresRecordIds, RecordChangedMessages message) throws IOException { - - jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds)); - List retryRecordInfos = new LinkedList<>(); - for (String recordId : failuresRecordIds) { - for (RecordInfo origMessage : recordInfos) { - if (origMessage.getId().equalsIgnoreCase(recordId)) { - retryRecordInfos.add(origMessage); - } - } - } + private Map getSourceMap(RecordIndexerPayload.Record record) { - RecordChangedMessages newMessage = RecordChangedMessages.builder() - .messageId(message.getMessageId()) - .publishTime(message.getPublishTime()) - .data(this.gson.toJson(retryRecordInfos)) - .attributes(message.getAttributes()).build(); + Map indexerPayload = new HashMap<>(); - String payLoad = this.gson.toJson(newMessage); - this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers); + // get the key and get the corresponding object from the individualRecord object + if (record.getData() != null) { + Map data = new HashMap<>(); + for (Map.Entry entry : record.getData().entrySet()) { + data.put(entry.getKey(), entry.getValue()); + } + indexerPayload.put(Constants.DATA, data); } - private void updateAuditLog() { - logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail); - logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail); - logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail); - logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail); + indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId()); + indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind()); + indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace()); + indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType()); + indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion()); + indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl()); + indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl())); + indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal()); + indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress()); + if (record.getAncestry() != null) { + indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry()); } - - private void logAuditEvents(OperationType operationType, Consumer> successEvent, Consumer> failedEvent) { - List succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType); - if(!succeededRecords.isEmpty()) { - successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList())); - } - List skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType); - List failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType); - failedRecords.addAll(skippedRecords); - if(!failedRecords.isEmpty()) { - failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList())); + return indexerPayload; + } + + private void retryAndEnqueueFailedRecords(List recordInfos, + List failuresRecordIds, RecordChangedMessages message) throws IOException { + + jaxRsDpsLog.info(String.format( + "queuing bulk failed records back to task-queue for retry | count: %s | records: %s", + failuresRecordIds.size(), failuresRecordIds)); + List retryRecordInfos = new LinkedList<>(); + for (String recordId : failuresRecordIds) { + for (RecordInfo origMessage : recordInfos) { + if (origMessage.getId().equalsIgnoreCase(recordId)) { + retryRecordInfos.add(origMessage); } + } + } + + RecordChangedMessages newMessage = RecordChangedMessages.builder() + .messageId(message.getMessageId()) + .publishTime(message.getPublishTime()) + .data(this.gson.toJson(retryRecordInfos)) + .attributes(message.getAttributes()).build(); + + String payLoad = this.gson.toJson(newMessage); + this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers); + } + + private void updateAuditLog() { + logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, + this.auditLogger::indexCreateRecordFail); + logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, + this.auditLogger::indexUpdateRecordFail); + logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, + this.auditLogger::indexPurgeRecordFail); + logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, + this.auditLogger::indexDeleteRecordFail); + } + + private void logAuditEvents(OperationType operationType, Consumer> successEvent, + Consumer> failedEvent) { + List succeededRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.SUCCESS, operationType); + if (!succeededRecords.isEmpty()) { + successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage) + .collect(Collectors.toList())); + } + List skippedRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.SKIP, operationType); + List failedRecords = this.jobStatus + .getRecordStatuses(IndexingStatus.FAIL, operationType); + failedRecords.addAll(skippedRecords); + if (!failedRecords.isEmpty()) { + failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage) + .collect(Collectors.toList())); } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java index 0e3c2016..86fb4344 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,13 @@ package org.opengroup.osdu.indexer.util; +import com.google.common.base.Strings; import java.util.Collections; import java.util.Map; import java.util.stream.Collectors; - -import javax.servlet.http.HttpServletRequest; import javax.inject.Inject; - -import com.google.common.base.Strings; +import javax.servlet.http.HttpServletRequest; import org.opengroup.osdu.core.common.model.http.DpsHeaders; - import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; import org.opengroup.osdu.core.gcp.util.TraceIdExtractor; import org.springframework.context.annotation.Primary; @@ -38,23 +35,23 @@ import org.springframework.web.context.annotation.RequestScope; @Primary public class DpsHeaderFactoryGcp extends DpsHeaders { - @Inject - public DpsHeaderFactoryGcp(HttpServletRequest request) { + @Inject + public DpsHeaderFactoryGcp(HttpServletRequest request) { - Map headers = Collections - .list(request.getHeaderNames()) - .stream() - .collect(Collectors.toMap(h -> h, request::getHeader)); + Map headers = Collections + .list(request.getHeaderNames()) + .stream() + .collect(Collectors.toMap(h -> h, request::getHeader)); - String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); + String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT); - if(!Strings.isNullOrEmpty(traceContext)){ - headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); - } + if (!Strings.isNullOrEmpty(traceContext)) { + headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext)); + } - this.addFromMap(headers); + this.addFromMap(headers); - // Add Correlation ID if missing - this.addCorrelationIdIfMissing(); - } + // Add Correlation ID if missing + this.addCorrelationIdIfMissing(); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java index b60ef1da..8e53b593 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java @@ -23,9 +23,10 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.indexer.config.MongoDBConfigProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component @@ -36,16 +37,8 @@ public class MongoClientHandler { private static final String MONGO_PREFIX = "mongodb://"; private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; - private MongoClient mongoClient = null; - - @Value("${mongo.db.url:#{null}}") - private String dbUrl; - - @Value("${mongo.db.user:#{null}}") - private String dbUser; - - @Value("${mongo.db.password:#{null}}") - private String dbPassword; + private com.mongodb.client.MongoClient mongoClient = null; + private MongoDBConfigProperties mongoDBConfigProperties; private MongoClient getOrInitMongoClient() throws RuntimeException { if (mongoClient != null) { @@ -54,9 +47,9 @@ public class MongoClientHandler { final String connectionString = String.format("%s%s:%s@%s/?%s", MONGO_PREFIX, - dbUser, - dbPassword, - dbUrl, + mongoDBConfigProperties.getMongoDbUser(), + mongoDBConfigProperties.getMongoDbPassword(), + mongoDBConfigProperties.getMongoDbUrl(), MONGO_OPTIONS); ConnectionString connString = new ConnectionString(connectionString); MongoClientSettings settings = MongoClientSettings.builder() @@ -80,4 +73,8 @@ public class MongoClientHandler { return mongoClient; } + @Autowired + public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) { + this.mongoDBConfigProperties = mongoDBConfigProperties; + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java index 2396dd1e..1e3c3f24 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,101 +17,108 @@ package org.opengroup.osdu.indexer.util; +import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; + import com.google.common.base.Strings; +import java.util.Map; import lombok.extern.java.Log; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; import org.opengroup.osdu.core.gcp.model.AppEngineHeaders; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; -import javax.inject.Inject; -import java.util.Map; - -import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION; - - @Log @Component @RequestScope public class RequestInfoImpl implements IRequestInfo { - @Inject - private DpsHeaders dpsHeaders; - - @Inject - private IServiceAccountJwtClient serviceAccountJwtClient; - - @Inject - private TenantInfo tenantInfo; - - @Value("${DEPLOYMENT_ENVIRONMENT}") - private String DEPLOYMENT_ENVIRONMENT; - - private static final String expectedCronHeaderValue = "true"; - - @Override - public DpsHeaders getHeaders() { - - return this.dpsHeaders; - } - - @Override - public String getPartitionId() { - return this.dpsHeaders.getPartitionId(); - } - - @Override - public Map getHeadersMap() { - return this.dpsHeaders.getHeaders(); - } - - @Override - public Map getHeadersMapWithDwdAuthZ() { - return getHeadersWithDwdAuthZ().getHeaders(); - } - - @Override - public DpsHeaders getHeadersWithDwdAuthZ() { - // Update DpsHeaders so that service account creds are passed down - this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader()); - return this.dpsHeaders; - } - - @Override - public boolean isCronRequest() { - String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null); - return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); - } - - @Override - public boolean isTaskQueueRequest() { - if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false; - - String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); - return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final TenantInfo tenantInfo; + private final IServiceAccountJwtClient serviceAccountJwtClient; + private final DpsHeaders dpsHeaders; + + @Autowired + public RequestInfoImpl(IndexerConfigurationProperties indexerConfigurationProperties, + TenantInfo tenantInfo, IServiceAccountJwtClient serviceAccountJwtClient, + DpsHeaders dpsHeaders) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.tenantInfo = tenantInfo; + this.serviceAccountJwtClient = serviceAccountJwtClient; + this.dpsHeaders = dpsHeaders; + } + + private static final String expectedCronHeaderValue = "true"; + + @Override + public DpsHeaders getHeaders() { + + return this.dpsHeaders; + } + + @Override + public String getPartitionId() { + return this.dpsHeaders.getPartitionId(); + } + + @Override + public Map getHeadersMap() { + return this.dpsHeaders.getHeaders(); + } + + @Override + public Map getHeadersMapWithDwdAuthZ() { + return getHeadersWithDwdAuthZ().getHeaders(); + } + + @Override + public DpsHeaders getHeadersWithDwdAuthZ() { + // Update DpsHeaders so that service account creds are passed down + this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader()); + return this.dpsHeaders; + } + + @Override + public boolean isCronRequest() { + String appEngineCronHeader = this.dpsHeaders.getHeaders() + .getOrDefault(AppEngineHeaders.CRON_SERVICE, null); + return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader); + } + + @Override + public boolean isTaskQueueRequest() { + if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) { + return false; } - public String checkOrGetAuthorizationHeader() { - if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) { - String authHeader = this.dpsHeaders.getAuthorization(); - if (Strings.isNullOrEmpty(authHeader)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty"); - } - String user = this.dpsHeaders.getUserEmail(); - if (Strings.isNullOrEmpty(user)) { - throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty"); - } - return authHeader; - } else { - return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); - } + String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME); + return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); + } + + public String checkOrGetAuthorizationHeader() { + if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment()) + == DeploymentEnvironment.LOCAL) { + String authHeader = this.dpsHeaders.getAuthorization(); + if (Strings.isNullOrEmpty(authHeader)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", + "Authorization token cannot be empty"); + } + String user = this.dpsHeaders.getUserEmail(); + if (Strings.isNullOrEmpty(user)) { + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", + "User header cannot be empty"); + } + return authHeader; + } else { + return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName()); } + } } diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java index 6da39c9e..98605a15 100644 --- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java +++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,12 +30,10 @@ import com.google.api.services.iam.v1.model.SignJwtRequest; import com.google.api.services.iam.v1.model.SignJwtResponse; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import java.io.FileInputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.inject.Inject; import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; @@ -55,7 +53,8 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; @@ -63,134 +62,141 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient { - private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; - private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; - - private final JsonFactory JSON_FACTORY = new JacksonFactory(); - - private Iam iam; - - @Inject - private ITenantFactory tenantInfoServiceProvider; - @Inject - private IJwtCache cacheService; - @Inject - private JaxRsDpsLog log; - @Inject - private DpsHeaders dpsHeaders; - - @Value("${GOOGLE_AUDIENCES}") - public String GOOGLE_AUDIENCES; - - @Value("${INDEXER_HOST}") - public String INDEXER_HOST; - - public String getIdToken(String tenantName) { - this.log.info("Tenant name received for auth token is: " + tenantName); - TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); - if (tenant == null) { - this.log.error("Invalid tenant name receiving from pubsub"); - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub"); - } - try { - - IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); - // Add the user to DpsHeaders directly - this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); - - if (!IdToken.refreshToken(cachedToken)) { - return cachedToken.getTokenValue(); - } - - // Getting signed JWT - Map signJwtPayload = this.getJWTCreationPayload(tenant); - - SignJwtRequest signJwtRequest = new SignJwtRequest(); - signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); - - String serviceAccountName = String - .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); - - Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() - .signJwt(serviceAccountName, signJwtRequest); - SignJwtResponse signJwtResponse = signJwt.execute(); - String signedJwt = signJwtResponse.getSignedJwt(); - - // Getting id token - List postParameters = new ArrayList<>(); - postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); - postParameters.add(new BasicNameValuePair("assertion", signedJwt)); - - HttpPost post = new HttpPost(JWT_AUDIENCE); - post.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); - post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); - - try (CloseableHttpClient httpclient = HttpClients.createDefault(); - CloseableHttpResponse httpResponse = httpclient.execute(post)) { - JsonObject jsonContent = new JsonParser().parse(EntityUtils.toString(httpResponse.getEntity())) - .getAsJsonObject(); - - if (!jsonContent.has("id_token")) { - log.error(String.format("Google IAM response: %s", jsonContent.toString())); - throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", - "The user is not authorized to perform this action"); - } - - String token = jsonContent.get("id_token").getAsString(); - IdToken idToken = IdToken.builder().tokenValue(token) - .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); - - this.cacheService.put(tenant.getServiceAccount(), idToken); - - return token; - } - } catch (JWTDecodeException e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", - "Invalid token, error decoding", e); - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", - e); - } - } - - public Iam getIam() throws Exception { - - if (this.iam == null) { - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - - // Authenticate using Google Application Default Credentials. - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - if (credential.createScopedRequired()) { - List scopes = new ArrayList<>(); - // Enable full Cloud Platform scope. - scopes.add(IamScopes.CLOUD_PLATFORM); - credential = credential.createScoped(scopes); - } - - // Create IAM API object associated with the authenticated transport. - this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(INDEXER_HOST) - .build(); - } - - return this.iam; - } - - private Map getJWTCreationPayload(TenantInfo tenantInfo) { - - Map payload = new HashMap<>(); - String googleAudience = GOOGLE_AUDIENCES; - if (googleAudience.contains(",")) { - googleAudience = googleAudience.split(",")[0]; - } - payload.put("target_audience", googleAudience); - payload.put("exp", System.currentTimeMillis() / 1000 + 3600); - payload.put("iat", System.currentTimeMillis() / 1000); - payload.put("iss", tenantInfo.getServiceAccount()); - payload.put("aud", JWT_AUDIENCE); - - return payload; - } + private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token"; + private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s"; + + private final JsonFactory JSON_FACTORY = new JacksonFactory(); + private final IndexerConfigurationProperties indexerConfigurationProperties; + private final ITenantFactory tenantInfoServiceProvider; + private final IJwtCache cacheService; + private final JaxRsDpsLog log; + private final DpsHeaders dpsHeaders; + + private Iam iam; + + @Autowired + public ServiceAccountJwtGcpClientImpl( + IndexerConfigurationProperties indexerConfigurationProperties, + ITenantFactory tenantInfoServiceProvider, + IJwtCache cacheService, JaxRsDpsLog log, DpsHeaders dpsHeaders) { + this.indexerConfigurationProperties = indexerConfigurationProperties; + this.tenantInfoServiceProvider = tenantInfoServiceProvider; + this.cacheService = cacheService; + this.log = log; + this.dpsHeaders = dpsHeaders; + } + + public String getIdToken(String tenantName) { + this.log.info("Tenant name received for auth token is: " + tenantName); + TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName); + if (tenant == null) { + this.log.error("Invalid tenant name receiving from pubsub"); + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", + "Invalid tenant Name from pubsub"); + } + try { + + IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount()); + // Add the user to DpsHeaders directly + this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount()); + + if (!IdToken.refreshToken(cachedToken)) { + return cachedToken.getTokenValue(); + } + + // Getting signed JWT + Map signJwtPayload = this.getJWTCreationPayload(tenant); + + SignJwtRequest signJwtRequest = new SignJwtRequest(); + signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload)); + + String serviceAccountName = String + .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount()); + + Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts() + .signJwt(serviceAccountName, signJwtRequest); + SignJwtResponse signJwtResponse = signJwt.execute(); + String signedJwt = signJwtResponse.getSignedJwt(); + + // Getting id token + List postParameters = new ArrayList<>(); + postParameters + .add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); + postParameters.add(new BasicNameValuePair("assertion", signedJwt)); + + HttpPost post = new HttpPost(JWT_AUDIENCE); + post.setHeader(HttpHeaders.CONTENT_TYPE, + ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); + post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); + + try (CloseableHttpClient httpclient = HttpClients.createDefault(); + CloseableHttpResponse httpResponse = httpclient.execute(post)) { + JsonObject jsonContent = new JsonParser() + .parse(EntityUtils.toString(httpResponse.getEntity())) + .getAsJsonObject(); + + if (!jsonContent.has("id_token")) { + log.error(String.format("Google IAM response: %s", jsonContent.toString())); + throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", + "The user is not authorized to perform this action"); + } + + String token = jsonContent.get("id_token").getAsString(); + IdToken idToken = IdToken.builder().tokenValue(token) + .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build(); + + this.cacheService.put(tenant.getServiceAccount(), idToken); + + return token; + } + } catch (JWTDecodeException e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", + "Invalid token, error decoding", e); + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", + "Error generating token", + e); + } + } + + public Iam getIam() throws Exception { + + if (this.iam == null) { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + + // Authenticate using Google Application Default Credentials. + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + if (credential.createScopedRequired()) { + List scopes = new ArrayList<>(); + // Enable full Cloud Platform scope. + scopes.add(IamScopes.CLOUD_PLATFORM); + credential = credential.createScoped(scopes); + } + + // Create IAM API object associated with the authenticated transport. + this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(indexerConfigurationProperties.getIndexerHost()) + .build(); + } + + return this.iam; + } + + private Map getJWTCreationPayload(TenantInfo tenantInfo) { + + Map payload = new HashMap<>(); + String googleAudience = indexerConfigurationProperties.getGoogleAudiences(); + if (googleAudience.contains(",")) { + googleAudience = googleAudience.split(",")[0]; + } + payload.put("target_audience", googleAudience); + payload.put("exp", System.currentTimeMillis() / 1000 + 3600); + payload.put("iat", System.currentTimeMillis() / 1000); + payload.put("iss", tenantInfo.getServiceAccount()); + payload.put("aud", JWT_AUDIENCE); + + return payload; + } } diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties index 09ea204d..ea75c38d 100644 --- a/provider/indexer-reference/src/main/resources/application-dev.properties +++ b/provider/indexer-reference/src/main/resources/application-dev.properties @@ -17,24 +17,28 @@ GOOGLE_CLOUD_PROJECT=nice-etching-277309 -INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com -STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= -STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas -STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch -STORAGE_RECORDS_BATCH_SIZE=20 +indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com -INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue +storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +storage-records-batch-size=20 -AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 -CRS_API=example.com +indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue + +authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +crs-api=example.com ## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -REDIS_SEARCH_HOST=127.0.0.1 +redis-group-host=127.0.0.1 +redis-search-host=127.0.0.1 -GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com +google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com -mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672 diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties index 4d62d4ab..8690c57a 100644 --- a/provider/indexer-reference/src/main/resources/application.properties +++ b/provider/indexer-reference/src/main/resources/application.properties @@ -1,6 +1,6 @@ # -# Copyright 2020 Google LLC -# Copyright 2020 EPAM Systems, Inc +# Copyright 2021 Google LLC +# Copyright 2021 EPAM Systems, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,56 +23,54 @@ server.port=8080 JAVA_OPTS=-Xms3072m -Xmx3072m JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 -DEPLOYMENT_ENVIRONMENT=CLOUD +environment=CLOUD -REDIS_GROUP_PORT=6379 -REDIS_SEARCH_PORT=6379 -DEFAULT_DATA_COUNTRY=US +#default cache settings +schema-cache-expiration=60 +index-cache-expiration=60 +elastic-cache-expiration=1440 +cursor-cache-expiration=60 +# kinds cache expiration 2*24*60 +kinds-cache-expiration=2880 +# attributes cache expiration 2*24*60 +attributes-cache-expiration=2880 -#Default Cache Settings -SCHEMA_CACHE_EXPIRATION=60 -INDEX_CACHE_EXPIRATION=60 -ELASTIC_CACHE_EXPIRATION=1440 -CURSOR_CACHE_EXPIRATION=60 -# Kinds Cache expiration 2*24*60 -KINDS_CACHE_EXPIRATION=2880 -# Attributes Cache expiration 2*24*60 -ATTRIBUTES_CACHE_EXPIRATION=2880 +kinds-redis-database=1 +cron-index-cleanup-threshold-days=3 +cron-empty-index-cleanup-threshold-days=7 -KINDS_REDIS_DATABASE=1 -CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 -CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 +gae-service=indexer +key-ring=csqp +kms-key=searchservice -GAE_SERVICE=indexer -KEY_RING=csqp -KMS_KEY=searchService +elastic-datastore-kind=searchsettings +elastic-datastore-id=indexer-service -ELASTIC_DATASTORE_KIND=SearchSettings -ELASTIC_DATASTORE_ID=indexer-service +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= -mongo.db.url=localhost:27017 -mongo.db.user= -mongo.db.password= +indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com +storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com -INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com -STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com +storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas +storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records +storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch +storage-records-batch-size=20 -STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas -STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch -STORAGE_RECORDS_BATCH_SIZE=20 +indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue -INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue - -AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 -LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 -CRS_API=example.com +authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1 +legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 +crs-api=example.com ## use below values for gcp: opendes -REDIS_GROUP_HOST=127.0.0.1 -REDIS_SEARCH_HOST=127.0.0.1 +redis-group-host=127.0.0.1 +redis-search-host=127.0.0.1 + +google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672 -GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com -mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672 diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java index 7ebfab8d..9ec5fb7a 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,12 @@ package org.opengroup.osdu.indexer.middleware; +import java.io.IOException; +import java.util.Collections; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -25,46 +31,51 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.util.Collections; - @RunWith(MockitoJUnitRunner.class) public class IndexFilterTest { - @InjectMocks - private IndexFilter indexFilter; + @InjectMocks + private IndexFilter indexFilter; - @Mock - private DpsHeaders dpsHeaders; + @Mock + private DpsHeaders dpsHeaders; - @Test - public void shouldSetCorrectResponseHeaders() throws IOException, ServletException { - HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class); - HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class); - FilterChain filterChain = Mockito.mock(FilterChain.class); - Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com"); - Mockito.when(httpServletRequest.getMethod()).thenReturn("POST"); - Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value"); + @Test + public void shouldSetCorrectResponseHeaders() throws IOException, ServletException { + HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class); + HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class); + FilterChain filterChain = Mockito.mock(FilterChain.class); + Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com"); + Mockito.when(httpServletRequest.getMethod()).thenReturn("POST"); + Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value"); - indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); + indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString()); - Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString()); - Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString()); - Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString()); - Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString()); - Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString()); - Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString()); - Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString()); - Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString()); - Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value"); - Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse); - } + Mockito.verify(httpServletResponse) + .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections + .singletonList( + "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey") + .toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", + Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString()); + Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", + Collections.singletonList("true").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString()); + Mockito.verify(httpServletResponse) + .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString()); + Mockito.verify(httpServletResponse).addHeader("Cache-Control", + Collections.singletonList("no-cache, no-store, must-revalidate").toString()); + Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", + Collections.singletonList("default-src 'self'").toString()); + Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", + Collections.singletonList("max-age=31536000; includeSubDomains").toString()); + Mockito.verify(httpServletResponse) + .addHeader("Expires", Collections.singletonList("0").toString()); + Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value"); + Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse); + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java index 9633d128..e1c53edb 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,24 @@ package org.opengroup.osdu.indexer.service; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.google.common.collect.Lists; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import org.elasticsearch.client.RestHighLevelClient; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.IndexInfo; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.core.common.search.IndicesService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; @@ -34,107 +42,111 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.mockito.Mockito.*; - @RunWith(SpringRunner.class) @PrepareForTest({RestHighLevelClient.class}) public class CronServiceImplTest { - @Mock - private RestHighLevelClient restHighLevelClient; - @Mock - private IndicesService indicesService; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private IRequestInfo requestInfo; - @Mock - private JaxRsDpsLog log; - @InjectMocks - private CronServiceImpl sut; - - @InjectMocks - private DpsHeaders dpsHeaders; - - @Before - public void setup() { - - when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); - - when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3); - when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3); - } - - @Test - public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception { - final String indexPattern = "tenant1-index-*"; - - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupIndices(indexPattern); - - verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern); - } - - @Test(expected = IOException.class) - public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception { - IOException exception = new IOException("blah"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception); - - this.sut.cleanupIndices("tenant1-test-*"); - - verify(this.indicesService, times(0)).deleteIndex(any(), any()); - } - - @Test - public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception { - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); - } - - @Test - public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception { - IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build(); - - when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info)); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); - verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); - } - - @Test(expected = IOException.class) - public void run_cleanup_when_backend_throws_exception() throws Exception { - IOException exception = new IOException("blah"); - when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); - when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception); - - this.sut.cleanupEmptyStaleIndices(); - - verify(this.indicesService, times(0)).deleteIndex(any(), any()); - } + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private IRequestInfo requestInfo; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private CronServiceImpl sut; + + @InjectMocks + private DpsHeaders dpsHeaders; + + @Before + public void setup() { + + when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders); + + when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3); + when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3); + } + + @Test + public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception { + final String indexPattern = "tenant1-index-*"; + + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10") + .creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupIndices(indexPattern); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")) + .thenThrow(exception); + + this.sut.cleanupIndices("tenant1-test-*"); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } + + @Test + public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10") + .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test + public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception { + IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0") + .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())) + .build(); + + when(this.requestInfo.getPartitionId()).thenReturn("tenant1"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)) + .thenReturn(Lists.newArrayList(info)); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0"); + verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null); + } + + @Test(expected = IOException.class) + public void run_cleanup_when_backend_throws_exception() throws Exception { + IOException exception = new IOException("blah"); + when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient); + when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception); + + this.sut.cleanupEmptyStaleIndices(); + + verify(this.indicesService, times(0)).deleteIndex(any(), any()); + } } \ No newline at end of file diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java index f9645a50..b09fbc40 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,101 +17,102 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; -import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.test.context.junit4.SpringRunner; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.when; - @RunWith(SpringRunner.class) public class ElasticSettingServiceTest { - @Mock - private ITenantInfoService tenantInfoService; - @Mock - private IElasticRepository elasticRepository; - @Mock - private IElasticCredentialsCache elasticCredentialCache; - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private TenantInfo tenantInfo; - @InjectMocks - private ElasticSettingServiceImpl sut; - @Mock - private ClusterSettings clusterSettings; - @Mock - private DpsHeaders headersInfo; + @Mock + private ITenantInfoService tenantInfoService; + @Mock + private IElasticRepository elasticRepository; + @Mock + private IElasticCredentialsCache elasticCredentialCache; + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private TenantInfo tenantInfo; + @InjectMocks + private ElasticSettingServiceImpl sut; + @Mock + private ClusterSettings clusterSettings; + @Mock + private DpsHeaders headersInfo; - @Mock - private JaxRsDpsLog log; + @Mock + private JaxRsDpsLog log; - public String GAE_SERVICE = "indexer"; + public String GAE_SERVICE = "indexer"; - private final String host = "db5c51c1.us-central1.gcp.cloud.es.io"; - private final int port = 9243; - private final String credentials = "name:password"; + private final String host = "db5c51c1.us-central1.gcp.cloud.es.io"; + private final int port = 9243; + private final String credentials = "name:password"; - String cacheKey = ""; + String cacheKey = ""; - @Before - public void setup() { - when(tenantInfo.getName()).thenReturn("tenant1"); - when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); - when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); - when(configurationProperties.getGaeService()).thenReturn("indexer"); - clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build(); - cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); - } + @Before + public void setup() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(this.headersInfo.getPartitionId()).thenReturn("tenant1"); + when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); + when(configurationProperties.getGaeService()).thenReturn("indexer"); + clusterSettings = ClusterSettings.builder().host(host).port(port) + .userNameAndPassword(credentials).build(); + cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName()); + } - @Test - public void should_getValid_clusterSettings_fromCache() { + @Test + public void should_getValid_clusterSettings_fromCache() { - when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); - ClusterSettings response = this.sut.getElasticClusterInformation(); - assertNotNull(response); - assertEquals(response.getHost(), host); - assertEquals(response.getPort(), port); - assertEquals(response.getUserNameAndPassword(), credentials); - } + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } - @Test - public void should_getValid_clusterSettings_fromCosmosDB() { + @Test + public void should_getValid_clusterSettings_fromCosmosDB() { - when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); + when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings); - when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings); + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings); - ClusterSettings response = this.sut.getElasticClusterInformation(); - assertNotNull(response); - assertEquals(response.getHost(), host); - assertEquals(response.getPort(), port); - assertEquals(response.getUserNameAndPassword(), credentials); - } + ClusterSettings response = this.sut.getElasticClusterInformation(); + assertNotNull(response); + assertEquals(response.getHost(), host); + assertEquals(response.getPort(), port); + assertEquals(response.getUserNameAndPassword(), credentials); + } - @Test(expected = AppException.class) - public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException { + @Test(expected = AppException.class) + public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException { - when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null); + when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null); - this.sut.getElasticClusterInformation(); + this.sut.getElasticClusterInformation(); - } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java index 9ef4a6ad..6a438da2 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,17 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; import org.apache.http.HttpEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; @@ -34,160 +42,162 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.indexer.logging.AuditLogger; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.logging.AuditLogger; import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) -@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class}) +@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, + EntityUtils.class}) public class IndexCopyServiceImplTest { - private final String correlationId = UUID.randomUUID().toString(); - - @Mock - private HttpEntity httpEntity; - @Mock - private HttpEntity httpEntityRequest; - @Mock - private IRequestInfo requestInfo; - @Mock - private DpsHeaders headersInfo; - @Mock - private RestClient restClient; - @Mock - private RestHighLevelClient restHighLevelClient; - @Mock - private IndicesService indicesService; - @Mock - private IndexerMappingService mappingService; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private ElasticIndexNameResolver elasticIndexNameResolver; - @Mock - private Response response; - @Mock - private IElasticSettingService elasticSettingService; - @Mock - private AuditLogger auditLogger; - @Mock - private Map httpHeaders; - @InjectMocks - private IndexCopyServiceImpl sut; - - private ClusterSettings commonCluster; - - private Map correctMap; - - @Before - public void setup() { - - commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build(); - - httpHeaders = new HashMap<>(); - httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); - httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); - when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); - when(response.getEntity()).thenReturn(httpEntity); - - Type mapType = new TypeToken>() {}.getType(); - String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}"; - correctMap = new Gson().fromJson(afterFormat, mapType); - - restHighLevelClient = mock(RestHighLevelClient.class); + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private HttpEntity httpEntity; + @Mock + private HttpEntity httpEntityRequest; + @Mock + private IRequestInfo requestInfo; + @Mock + private DpsHeaders headersInfo; + @Mock + private RestClient restClient; + @Mock + private RestHighLevelClient restHighLevelClient; + @Mock + private IndicesService indicesService; + @Mock + private IndexerMappingService mappingService; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private Response response; + @Mock + private IElasticSettingService elasticSettingService; + @Mock + private AuditLogger auditLogger; + @Mock + private Map httpHeaders; + @InjectMocks + private IndexCopyServiceImpl sut; + + private ClusterSettings commonCluster; + + private Map correctMap; + + @Before + public void setup() { + + commonCluster = ClusterSettings.builder().host("commonhost").port(8080) + .userNameAndPassword("username:pwd").build(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(response.getEntity()).thenReturn(httpEntity); + + Type mapType = new TypeToken>() { + }.getType(); + String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}"; + correctMap = new Gson().fromJson(afterFormat, mapType); + + restHighLevelClient = mock(RestHighLevelClient.class); + + } + + @Test(expected = IOException.class) + public void should_throwIOException_when_indexMappingNotFound() throws Exception { + IOException exception = new IOException( + "Fail to get mapping for the given index from common cluster."); + + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenThrow(exception); + + this.sut.copyIndex("common:metadata:entity:1.0.0"); + } + + @Test(expected = IllegalArgumentException.class) + public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() { + try { + this.sut.copyIndex(null); + } catch (IOException e) { + fail("Should not throw IOException but illegalArgumentException."); } - - @Test(expected = IOException.class) - public void should_throwIOException_when_indexMappingNotFound() throws Exception { - IOException exception = new IOException("Fail to get mapping for the given index from common cluster."); - - when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception); - - this.sut.copyIndex("common:metadata:entity:1.0.0"); + } + + @Test + public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() { + String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}"; + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + try { + when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(mappingJson); + Map resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test"); + Assert.assertEquals(resultMap, correctMap); + } catch (Exception ignored) { } + } - @Test(expected = IllegalArgumentException.class) - public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() { - try { - this.sut.copyIndex(null); - } catch (IOException e) { - fail("Should not throw IOException but illegalArgumentException."); - } - } + @Test + public void should_returnClusterInfo_getCommonClusterInformationtest() { + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; - @Test - public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() { - String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}"; - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - try { - when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson); - Map resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test"); - Assert.assertEquals(resultMap, correctMap); - } catch (Exception ignored) { - } - } + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - @Test - public void should_returnClusterInfo_getCommonClusterInformationtest() { - try { - String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster); - - String[] resultCommonCluster = this.sut.getCommonClusterInformation(); - Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]); - Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]); - Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]); - } catch (IOException ignored) { - fail("Should not throw this exception " + ignored.getMessage()); - } + String[] resultCommonCluster = this.sut.getCommonClusterInformation(); + Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]); + Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]); + Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); } - - @Test(expected = AppException.class) - public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() { - try { - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); - this.sut.createIndexInTenantCluster("test", "test", "test", correctMap); - } catch (IOException ignored) { - fail("Should not throw this exception " + ignored.getMessage()); - } + } + + @Test(expected = AppException.class) + public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() { + try { + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService + .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + this.sut.createIndexInTenantCluster("test", "test", "test", correctMap); + } catch (IOException ignored) { + fail("Should not throw this exception " + ignored.getMessage()); } - - @Ignore - public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() { - //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest(). - try { - String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; - Request request = new Request("POST", "/_reindex?wait_for_completion=false"); - request.setEntity(httpEntityRequest); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); - when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient); - when(restClient.performRequest(request)).thenReturn(response); - when(response.getEntity()).thenReturn(httpEntity); - Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster)); - } catch (IOException ignored) { - } + } + + @Ignore + public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() { + //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest(). + try { + String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"}; + Request request = new Request("POST", "/_reindex?wait_for_completion=false"); + request.setEntity(httpEntityRequest); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + when(indicesService + .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient); + when(restClient.performRequest(request)).thenReturn(response); + when(response.getEntity()).thenReturn(httpEntity); + Assert.assertEquals(httpEntity, + this.sut.reindexInTenantCluster("test", "test", correctCommonCluster)); + } catch (IOException ignored) { } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java deleted file mode 100644 index b4eca3a4..00000000 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.service; - -import com.google.gson.Gson; -import org.apache.http.StatusLine; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; -import org.elasticsearch.action.bulk.BulkItemResponse.Failure; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.*; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; -import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.indexer.util.TypeMapper; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.io.IOException; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.when; - -@Ignore -@RunWith(SpringRunner.class) -@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class}) -public class IndexerMappingServiceTest { - - private final String kind = "tenant:test:test:1.0.0"; - private final String index = "tenant-test-test-1.0.0"; - private final String type = "test"; - private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}"; - - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private RestClient restClient; - @Mock - private Response response; - @Mock - private StatusLine statusLine; - - @InjectMocks - private IndexerMappingServiceImpl sut; - - @Mock - private ElasticClientHandler elasticClientHandler; - - @InjectMocks - private RestHighLevelClient restHighLevelClient; - - @InjectMocks - private IndexSchema indexSchema; - @InjectMocks - private IndicesClient indicesClient; - - @InjectMocks - private AcknowledgedResponse mappingResponse; - - @Before - public void setup() throws IOException { - initMocks(this); - when(configurationProperties.isPreDemo()).thenReturn(true); - Map dataMapping = new HashMap<>(); - dataMapping.put("Location", "geo_point"); - dataMapping.put("Msg", "text"); - Map metaMapping = new HashMap<>(); - metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); - metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL)); - this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping) - .build(); - - this.indicesClient = PowerMockito.mock(IndicesClient.class); - this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class); - - when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient); - when(this.restClient.performRequest(any())).thenReturn(response); - when(this.response.getStatusLine()).thenReturn(statusLine); - when(this.statusLine.getStatusCode()).thenReturn(200); - } - - @Test - public void should_returnValidMapping_givenFalseMerge_createMappingTest() { - try { - String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false); - assertEquals(mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_returnValidMapping_givenTrueMerge_createMappingTest() { - try { - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - - String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); - assertEquals(this.mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_returnValidMapping_givenExistType_createMappingTest() { - try { - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - - IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl()); - doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any()); - String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); - assertEquals(this.mappingValid, mapping); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception { - try { - Set indices = new HashSet(); - indices.add("indices 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field"); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception { - try { - Set indices = new HashSet(); - indices.add("invalid 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (Exception e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception { - try { - Set indices = new HashSet(); - indices.add("indices 1"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid"); - } catch (Exception e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception { - try { - - Set indices = new HashSet(); - indices.add("indices 1"); - indices.add("indices Invalid"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException("")); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList()); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (AppException e) { - throw e; - } - } - - @Test(expected = AppException.class) - public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() { - try { - Set indices = new HashSet(); - indices.add("indices 1"); - indices.add("indices Invalid"); - GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); - doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.field("any field", new HashMap()); - builder.endObject(); - BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map mapBuilder = new HashMap<>(); - mapBuilder.put("data.any field", mappingMetaData); - Map> mappingBuilder = new HashMap<>(); - mappingBuilder.put("any index 1", mapBuilder); - mappingBuilder.put("any index 2", mapBuilder); - Map>> mapping = new HashMap<>(); - mapping.put("indices 1", mappingBuilder); - when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class)); - BulkByScrollResponse response = mock(BulkByScrollResponse.class); - doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class)); - when(response.getBulkFailures()).thenReturn(new ArrayList()); - when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException("")); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); - } catch (AppException e) { - throw e; - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - - @Test - public void should_returnDocumentMapping_givenValidIndexSchema() { - - try { - Map documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema); - String documentMappingJson = new Gson().toJson(documentMapping); - assertEquals(this.mappingValid, documentMappingJson); - - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } -} diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java index 27b5f786..4ce6eb4e 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,27 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; import org.apache.http.HttpStatus; import org.elasticsearch.client.RestHighLevelClient; import org.junit.Assert; @@ -25,380 +46,400 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.indexer.OperationType; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.RequestStatus; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl; import org.opengroup.osdu.indexer.util.ElasticClientHandler; -import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.mock; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) @PrepareForTest({RestHighLevelClient.class}) public class IndexerSchemaServiceTest { - private final String kind = "tenant:test:test:1.0.0"; - private final String emptySchema = null; - private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}"; - - @Mock - private JaxRsDpsLog log; - @Mock - private SchemaProviderImpl schemaProvider; - @Mock - private ElasticClientHandler elasticClientHandler; - @Mock - private ElasticIndexNameResolver elasticIndexNameResolver; - @Mock - private IndexerMappingService mappingService; - @Mock - private IndicesService indicesService; - @Mock - private ISchemaCache schemaCache; - @InjectMocks - private IndexSchemaServiceImpl sut; - - @Before - public void setup() { - initMocks(this); - RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class); - when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - } - - @Test - public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(emptySchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertNotNull(indexSchema); - } - - @Test - public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(someSchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertEquals(kind, indexSchema.getKind()); - } - - @Test - public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception { - when(schemaProvider.getSchema(any())).thenReturn(someSchema); - when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); - - IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); - - Assert.assertEquals(kind, indexSchema.getKind()); - } - - @Test - public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { - try { - String invalidSchema = "{}}"; - when(schemaProvider.getSchema(any())).thenReturn(invalidSchema); - - this.sut.getIndexerInputSchema(kind, false); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); - Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } - } - - @Test - public void should_return_basic_schema_when_storage_returns_no_schema() { - IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false); - - assertNotNull(returnedSchema.getDataSchema()); - assertNotNull(returnedSchema); - assertEquals(kind, returnedSchema.getKind()); - } - - @Test - public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"startDate\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"endDate\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"type \"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"itemguid\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); - verifyNoMoreInteractions(this.mappingService); - } - - @Test - public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"startDate\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any()); - verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean()); - verifyNoMoreInteractions(this.mappingService); - } - - @Test - public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); - - try { - this.sut.processSchemaMessages(schemaMessages); - } catch (AppException e) { - assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode()); - assertEquals("error creating or merging index mapping", e.getError().getMessage()); - assertEquals(reason, e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - } - - @Test - public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-")); - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); - - try { - this.sut.processSchemaMessages(schemaMessages); - } catch (AppException e) { - assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); - assertEquals("blah", e.getError().getMessage()); - assertEquals(reason, e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - } - - @Test - public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"" + - "}"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.create_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0")); + private final String kind = "tenant:test:test:1.0.0"; + private final String emptySchema = null; + private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}"; + + @Mock + private JaxRsDpsLog log; + @Mock + private SchemaProviderImpl schemaProvider; + @Mock + private ElasticClientHandler elasticClientHandler; + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + @Mock + private IndexerMappingService mappingService; + @Mock + private IndicesService indicesService; + @Mock + private ISchemaCache schemaCache; + @InjectMocks + private IndexSchemaServiceImpl sut; + + @Before + public void setup() { + initMocks(this); + RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class); + when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); + } + + @Test + public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(emptySchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertNotNull(indexSchema); + } + + @Test + public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() + throws Exception { + when(schemaProvider.getSchema(any())).thenReturn(someSchema); + when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema); + + IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false); + + Assert.assertEquals(kind, indexSchema.getKind()); + } + + @Test + public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() { + try { + String invalidSchema = "{}}"; + when(schemaProvider.getSchema(any())).thenReturn(invalidSchema); + + this.sut.getIndexerInputSchema(kind, false); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); + Assert.assertEquals("An error has occurred while normalizing the schema.", + e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } - - @Test - public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.purge_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.schemaCache.get(kind)).thenReturn("schema"); - when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema"); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.schemaCache, times(2)).get(anyString()); - verify(this.schemaCache, times(2)).delete(anyString()); + } + + @Test + public void should_return_basic_schema_when_storage_returns_no_schema() { + IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false); + + assertNotNull(returnedSchema.getDataSchema()); + assertNotNull(returnedSchema); + assertEquals(kind, returnedSchema.getKind()); + } + + @Test + public void should_create_schema_when_storage_returns_valid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"endDate\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"type \"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"itemguid\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_merge_mapping_when_storage_returns_valid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"startDate\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any()); + verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String + .format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())) + .thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, "")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode()); + assertEquals("error creating or merging index mapping", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - - @Test - public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - Map schemaMessages = new HashMap<>(); - schemaMessages.put(kind, OperationType.purge_schema); - - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - - this.sut.processSchemaMessages(schemaMessages); - - verify(this.log).warning(eq(String.format("Kind: %s not found", kind))); + } + + @Test + public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String reason = String + .format("Could not create type mapping %s/completion.", kind.replace(":", "-")); + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())) + .thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah")); + + try { + this.sut.processSchemaMessages(schemaMessages); + } catch (AppException e) { + assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals(reason, e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - - @Test - public void should_sync_schema_with_storage() throws Exception { - String kind = "tenant1:avocet:completion:1.0.0"; - String storageSchema = "{" + - " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"status\"," + - " \"kind\": \"string\"" + - " }" + - " ]" + - "}"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); - when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); - - this.sut.syncIndexMappingWithStorageSchema(kind); - - verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, times(1)).isIndexExist(any(), any()); - verify(this.indicesService, times(1)).deleteIndex(any(), any()); - verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); - verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_log_and_do_nothing_when_storage_returns_invalid_schema() + throws IOException, URISyntaxException { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"" + + "}"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.create_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0")); + } + + @Test + public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.schemaCache.get(kind)).thenReturn("schema"); + when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema"); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.schemaCache, times(2)).get(anyString()); + verify(this.schemaCache, times(2)).delete(anyString()); + } + + @Test + public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + Map schemaMessages = new HashMap<>(); + schemaMessages.put(kind, OperationType.purge_schema); + + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + this.sut.processSchemaMessages(schemaMessages); + + verify(this.log).warning(eq(String.format("Kind: %s not found", kind))); + } + + @Test + public void should_sync_schema_with_storage() throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + String storageSchema = "{" + + " \"kind\": \"tenant1:avocet:completion:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"status\"," + + " \"kind\": \"string\"" + + " }" + + " ]" + + "}"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())).thenReturn(true); + when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema); + + this.sut.syncIndexMappingWithStorageSchema(kind); + + verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any()); + verifyNoMoreInteractions(this.mappingService); + } + + @Test + public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() + throws Exception { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.schemaCache.get(kind)).thenReturn(null); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + when(this.indicesService.deleteIndex(any(), any())) + .thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah")); + + try { + this.sut.syncIndexMappingWithStorageSchema(kind); + } catch (AppException e) { + assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode()); + assertEquals("blah", e.getError().getMessage()); + assertEquals("Index deletion error", e.getError().getReason()); + } catch (Exception e) { + fail("Should not throw this exception " + e.getMessage()); } - @Test - public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.schemaCache.get(kind)).thenReturn(null); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah")); - - try { - this.sut.syncIndexMappingWithStorageSchema(kind); - } catch (AppException e) { - assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode()); - assertEquals("blah", e.getError().getMessage()); - assertEquals("Index deletion error", e.getError().getReason()); - } catch (Exception e) { - fail("Should not throw this exception " + e.getMessage()); - } - - verify(this.indicesService, times(1)).isIndexExist(any(), any()); - verify(this.indicesService, times(1)).deleteIndex(any(), any()); - verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any()); - verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any()); - } - - @Test - public void should_return_true_while_if_forceClean_requested() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - - assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true)); - } - - @Test - public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); - - assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false)); - } - - @Test - public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException { - String kind = "tenant1:avocet:completion:1.0.0"; - when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-")); - when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); - - assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false)); - } + verify(this.indicesService, times(1)).isIndexExist(any(), any()); + verify(this.indicesService, times(1)).deleteIndex(any(), any()); + verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any()); + verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any()); + } + + @Test + public void should_return_true_while_if_forceClean_requested() throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true)); + } + + @Test + public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(false); + + assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false)); + } + + @Test + public void should_return_false_while_if_forceClean_notRequested_and_indexExist() + throws IOException { + String kind = "tenant1:avocet:completion:1.0.0"; + when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)) + .thenReturn(kind.replace(":", "-")); + when(this.indicesService.isIndexExist(any(), any())).thenReturn(true); + + assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false)); + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java index 8cc2f75c..bf7fa01a 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,17 @@ package org.opengroup.osdu.indexer.service; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -35,119 +46,116 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; import org.springframework.test.context.junit4.SpringRunner; -import java.util.*; - -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.mockStatic; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(SpringRunner.class) public class ReindexServiceTest { - private final String cursor = "100"; - - private final String correlationId = UUID.randomUUID().toString(); - - @Mock - private IndexerConfigurationProperties configurationProperties; - @Mock - private StorageService storageService; - @Mock - private IRequestInfo requestInfo; - @Mock - private IndexerQueueTaskBuilder indexerQueueTaskBuilder; - @Mock - private JaxRsDpsLog log; - @InjectMocks - private ReindexServiceImpl sut; - - private RecordReindexRequest recordReindexRequest; - private RecordQueryResponse recordQueryResponse; - - private Map httpHeaders; - - @Before - public void setup() { - initMocks(this); - - mockStatic(UUID.class); - - recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build(); - recordQueryResponse = new RecordQueryResponse(); - - httpHeaders = new HashMap<>(); - httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); - httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); - DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders); - when(requestInfo.getHeaders()).thenReturn(standardHeaders); - when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); - when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders); + private final String cursor = "100"; + + private final String correlationId = UUID.randomUUID().toString(); + + @Mock + private IndexerConfigurationProperties configurationProperties; + @Mock + private StorageService storageService; + @Mock + private IRequestInfo requestInfo; + @Mock + private IndexerQueueTaskBuilder indexerQueueTaskBuilder; + @Mock + private JaxRsDpsLog log; + @InjectMocks + private ReindexServiceImpl sut; + + private RecordReindexRequest recordReindexRequest; + private RecordQueryResponse recordQueryResponse; + + private Map httpHeaders; + + @Before + public void setup() { + initMocks(this); + + mockStatic(UUID.class); + + recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0") + .cursor(cursor).build(); + recordQueryResponse = new RecordQueryResponse(); + + httpHeaders = new HashMap<>(); + httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth"); + httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId); + DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders); + when(requestInfo.getHeaders()).thenReturn(standardHeaders); + when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders); + when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders); + } + + @Test + public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(null); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String response = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } - @Test - public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setResults(null); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + @Test + public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setResults(new ArrayList<>()); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - String response = sut.reindexRecords(recordReindexRequest, false); + String response = sut.reindexRecords(recordReindexRequest, false); - Assert.assertNull(response); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } + Assert.assertNull(response); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } - @Test - public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setResults(new ArrayList<>()); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + @Ignore + @Test + public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() { + try { + recordQueryResponse.setCursor(cursor); + List results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); - String response = sut.reindexRecords(recordReindexRequest, false); + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1); - Assert.assertNull(response); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - @Ignore - @Test - public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() { - try { - recordQueryResponse.setCursor(cursor); - List results = new ArrayList<>(); - results.add("test1"); - recordQueryResponse.setResults(results); - - when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1); - - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); - Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } + Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", + taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } - - @Test - public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() { - try { - List results = new ArrayList<>(); - results.add("test1"); - recordQueryResponse.setResults(results); - when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); - - String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); - - Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload); - } catch (Exception e) { - fail("Should not throw exception" + e.getMessage()); - } + } + + @Test + public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() { + try { + List results = new ArrayList<>(); + results.add("test1"); + recordQueryResponse.setResults(results); + when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse); + + String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false); + + Assert.assertEquals(String.format( + "{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", + correlationId), taskQueuePayload); + } catch (Exception e) { + fail("Should not throw exception" + e.getMessage()); } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java index b4ea76b8..4b46abf2 100644 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java +++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java @@ -1,6 +1,6 @@ /* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,23 @@ package org.opengroup.osdu.indexer.service; +import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - +import java.lang.reflect.Type; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,227 +41,232 @@ import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.http.IUrlFetchService; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.http.HttpResponse; import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; import org.opengroup.osdu.core.common.model.indexer.JobStatus; import org.opengroup.osdu.core.common.model.indexer.RecordInfo; -import org.opengroup.osdu.core.common.model.http.HttpResponse; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; -import org.opengroup.osdu.core.common.http.IUrlFetchService; import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse; import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; import org.opengroup.osdu.core.common.model.indexer.Records; +import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; import org.springframework.http.HttpStatus; import org.springframework.test.context.junit4.SpringRunner; -import java.lang.reflect.Type; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import static java.util.Collections.singletonList; -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.powermock.api.mockito.PowerMockito.when; - @RunWith(SpringRunner.class) public class StorageServiceTest { - @Mock - private IUrlFetchService urlFetchService; - @Mock - private JobStatus jobStatus; - @Mock - private JaxRsDpsLog log; - @Mock - private IRequestInfo requestInfo; - @Mock - private IndexerConfigurationProperties configurationProperties; - @InjectMocks - private StorageServiceImpl sut; + @Mock + private IUrlFetchService urlFetchService; + @Mock + private JobStatus jobStatus; + @Mock + private JaxRsDpsLog log; + @Mock + private IRequestInfo requestInfo; + @Mock + private IndexerConfigurationProperties configurationProperties; + @InjectMocks + private StorageServiceImpl sut; - private List ids; - private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465"; - private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813"; + private List ids; + private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465"; + private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813"; - @Before - public void setup() { + @Before + public void setup() { - String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," + - "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]"; + String recordChangedMessages = + "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," + + + "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]"; - when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>()); - when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders()); + when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>()); + when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders()); - Type listType = new TypeToken>() {}.getType(); + Type listType = new TypeToken>() { + }.getType(); - List msgs = (new Gson()).fromJson(recordChangedMessages, listType); - jobStatus.initialize(msgs); - ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); + List msgs = (new Gson()).fromJson(recordChangedMessages, listType); + jobStatus.initialize(msgs); + ids = Arrays.asList(RECORD_ID1, RECORDS_ID2); - when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20); - } + when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20); + } - @Test - public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_return404_givenNullData_getValidStorageRecordsTest() + throws URISyntaxException { - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(null); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(null); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - should_return404_getValidStorageRecordsTest(); - } + should_return404_getValidStorageRecordsTest(); + } - @Test - public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_return404_givenEmptyData_getValidStorageRecordsTest() + throws URISyntaxException { - String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}"; + String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - should_return404_getValidStorageRecordsTest(); - } + should_return404_getValidStorageRecordsTest(); + } - @Test - public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException { + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() + throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}"; + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); - assertEquals(1, storageRecords.getRecords().size()); - } + assertEquals(1, storageRecords.getRecords().size()); + } - @Test - public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException { + @Test + public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}"; + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); - assertEquals(1, storageRecords.getRecords().size()); - verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813"); - } + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, + HttpStatus.NOT_FOUND.value(), + "Partial response received from Storage service - missing records", + "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813"); + } - @Test - public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException { - String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}"; + @Test + public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() + throws URISyntaxException { + String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}"; - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); - when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2)); + when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2)); - assertEquals(1, storageRecords.getRecords().size()); - verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed")); - } + assertEquals(1, storageRecords.getRecords().size()); + verify(this.jobStatus) + .addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), + "crs conversion failed", String + .format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", + "crs conversion failed")); + } - @Test - public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception { + @Test + public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() + throws Exception { - RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build(); + RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder() + .kind("tenant:test:test:1.0.0").cursor("100").build(); - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class)); + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class)); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest); + RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest); - assertEquals("100", recordQueryResponse.getCursor()); - assertNull(recordQueryResponse.getResults()); - } + assertEquals("100", recordQueryResponse.getCursor()); + assertNull(recordQueryResponse.getResults()); + } - @Test - public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception { - - String validSchemaFromStorage = "{" + - " \"kind\": \"tenant:test:test:1.0.0\"," + - " \"schema\": [" + - " {" + - " \"path\": \"msg\"," + - " \"kind\": \"string\"" + - " }," + - " {" + - " \"path\": \"references.entity\"," + - " \"kind\": \"string\"" + - " }" + - " ]," + - " \"ext\": null" + - "}"; - String kind = "tenant:test:test:1.0.0"; - - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setResponseCode(HttpStatus.OK.value()); - httpResponse.setBody(validSchemaFromStorage); - - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - - String recordSchemaResponse = this.sut.getStorageSchema(kind); - - assertNotNull(recordSchemaResponse); - } + @Test + public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception { - @Test - public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception { + String validSchemaFromStorage = "{" + + " \"kind\": \"tenant:test:test:1.0.0\"," + + " \"schema\": [" + + " {" + + " \"path\": \"msg\"," + + " \"kind\": \"string\"" + + " }," + + " {" + + " \"path\": \"references.entity\"," + + " \"kind\": \"string\"" + + " }" + + " ]," + + " \"ext\": null" + + "}"; + String kind = "tenant:test:test:1.0.0"; - String kind = "tenant:test:test:1.0.0"; + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.OK.value()); + httpResponse.setBody(validSchemaFromStorage); - HttpResponse httpResponse = new HttpResponse(); - httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value()); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + String recordSchemaResponse = this.sut.getStorageSchema(kind); - String recordSchemaResponse = this.sut.getStorageSchema(kind); + assertNotNull(recordSchemaResponse); + } - assertNull(recordSchemaResponse); - } + @Test + public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception { - @Test - public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException { + String kind = "tenant:test:test:1.0.0"; - String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}"; + HttpResponse httpResponse = new HttpResponse(); + httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value()); - HttpResponse httpResponse = mock(HttpResponse.class); - Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); - Records storageRecords = this.sut.getStorageRecords(ids); + String recordSchemaResponse = this.sut.getStorageSchema(kind); - assertEquals(1, storageRecords.getRecords().size()); + assertNull(recordSchemaResponse); + } - assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size()); + @Test + public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() + throws URISyntaxException { - assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0)); - } + String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}"; + + HttpResponse httpResponse = mock(HttpResponse.class); + Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage); + + when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse); + Records storageRecords = this.sut.getStorageRecords(ids); + + assertEquals(1, storageRecords.getRecords().size()); + + assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size()); + + assertEquals("conversion error occurred", + storageRecords.getConversionStatuses().get(0).getErrors().get(0)); + } - private void should_return404_getValidStorageRecordsTest() { - try { - this.sut.getStorageRecords(ids); - fail("Should throw exception"); - } catch (AppException e) { - assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } + private void should_return404_getValidStorageRecordsTest() { + try { + this.sut.getStorageRecords(ids); + fail("Should throw exception"); + } catch (AppException e) { + assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); } + } } diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java deleted file mode 100644 index 2c681985..00000000 --- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2020 Google LLC - * Copyright 2020 EPAM Systems, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.util; - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.services.iam.v1.Iam; -import com.google.api.services.iam.v1.model.SignJwtResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Spy; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment; -import org.opengroup.osdu.core.common.model.search.IdToken; -import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache; -import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.MockitoAnnotations.initMocks; -import static org.powermock.api.mockito.PowerMockito.when; - -@Ignore -@RunWith(SpringRunner.class) -@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, IndexerConfigurationProperties.class}) -public class ServiceAccountJwtGcpClientImplTest { - - private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw"; - - @Mock - private IndexerConfigurationProperties indexerConfigurationProperties; - @Mock - private JaxRsDpsLog log; - @Mock - private GoogleCredential credential; - @Mock - private NetHttpTransport httpTransport; - @Mock - private SignJwtResponse signJwtResponse; - @Mock - private Iam iam; - @Mock - private Iam.Projects iamProject; - @Mock - private Iam.Projects.ServiceAccounts iamProjectServiceAccounts; - @Mock - private Iam.Projects.ServiceAccounts.SignJwt signJwt; - @Mock - private CloseableHttpClient httpClient; - @Mock - private CloseableHttpResponse httpResponse; -// @InjectMocks -// private TenantInfoServiceImpl tenantInfoServiceProvider; -// @Mock -// private TenantInfoServiceImpl tenantInfoService; - @Mock - private IJwtCache cacheService; - @InjectMocks @Spy - private ServiceAccountJwtGcpClientImpl sut; - @Before - public void setup() throws Exception { - initMocks(this); - -// mockStatic(GoogleNetHttpTransport.class); -// mockStatic(GoogleCredential.class); -// mockStatic(HttpClients.class); -// mockStatic(EntityUtils.class); -// mockStatic(Config.class); - - when(GoogleNetHttpTransport.newTrustedTransport()).thenReturn(httpTransport); - when(GoogleCredential.getApplicationDefault()).thenReturn(credential); - when(credential.createScopedRequired()).thenReturn(true); - when(credential.createScoped(any())).thenReturn(credential); - when(HttpClients.createDefault()).thenReturn(httpClient); - when(httpClient.execute(any())).thenReturn(httpResponse); - when(indexerConfigurationProperties.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL); - when(indexerConfigurationProperties.getGoogleAudiences()).thenReturn("aud"); - -// when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService); - - TenantInfo tenantInfo = new TenantInfo(); - tenantInfo.setServiceAccount("tenant"); -// when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo); - - when(this.sut.getIam()).thenReturn(iam); - when(this.iam.projects()).thenReturn(iamProject); - when(this.iamProject.serviceAccounts()).thenReturn(iamProjectServiceAccounts); - when(this.iamProjectServiceAccounts.signJwt(any(), any())).thenReturn(signJwt); - when(this.signJwt.execute()).thenReturn(signJwtResponse); - when(this.signJwtResponse.getSignedJwt()).thenReturn("testJwt"); - - } - - @Test - public void should_returnCachedToken_givenCachedToken_getIdTokenTest() { - String tokenValue = "tokenValue"; - IdToken idToken = IdToken.builder().tokenValue(tokenValue).expirationTimeMillis(System.currentTimeMillis() + 10000000L).build(); - when(this.cacheService.get(any())).thenReturn(idToken); - - String returnedIdToken = this.sut.getIdToken(tokenValue); - - Assert.assertEquals(tokenValue, returnedIdToken); - } - - @Test - public void should_returnValidToken_getIdTokenTest() throws Exception { - when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", JWT_TOKEN)); - - String returnedToken = this.sut.getIdToken("tenant"); - - Assert.assertEquals(JWT_TOKEN, returnedToken); - } - - @Test - public void should_return500_given_invalidJWTResponse_getIdTokenException() { - try { - when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", "invalid jwt")); - - this.sut.getIdToken("tenant"); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode()); - Assert.assertEquals("Invalid token, error decoding", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } - - @Test - public void should_return403_given_missingIdTokenResponse_getIdTokenException() { - try { - when(EntityUtils.toString(any())).thenReturn("{}"); - - this.sut.getIdToken("tenant"); - fail("Should throw exception"); - } catch (AppException e) { - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode()); - Assert.assertEquals("The user is not authorized to perform this action", e.getError().getMessage()); - } catch (Exception e) { - fail("Should not throw this exception" + e.getMessage()); - } - } -} -- GitLab From bf2864b617597f9caf7c064a4f940adc8a7f2754 Mon Sep 17 00:00:00 2001 From: Sviatoslav Nekhaienko Date: Mon, 18 Jan 2021 16:23:41 +0200 Subject: [PATCH 08/11] allOf has been improved --- .../schema/converter/PropertiesProcessor.java | 23 +++++++- .../converter/SchemaToStorageFormatImpl.java | 5 ++ .../schema/converter/tags/AllOfItem.java | 2 + .../schema/converter/tags/Definition.java | 2 + .../schema/converter/tags/PropertiesData.java | 2 + .../schema/converter/tags/TypeProperty.java | 2 + .../SchemaToStorageFormatImplTest.java | 31 ++++++++-- .../converter/root-properties/schema.json | 12 ++++ .../converter/root-properties/schema.json.res | 9 +++ .../tags/allOf/allOf-inside-allOf.json | 56 +++++++++++++++++++ .../tags/allOf/allOf-inside-allOf.json.res | 21 +++++++ .../tags/allOf/allOf-inside-property.json | 33 +++++++++++ .../tags/allOf/allOf-inside-property.json.res | 14 +++++ .../converter/tags/allOf/indefinitions.json | 29 ++++++++++ .../tags/allOf/indefinitions.json.res | 9 +++ pom.xml | 2 +- 16 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 indexer-core/src/test/resources/converter/root-properties/schema.json create mode 100644 indexer-core/src/test/resources/converter/root-properties/schema.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json create mode 100644 indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 6de10d52..3506c2ca 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -56,6 +56,10 @@ public class PropertiesProcessor { public Stream> processItem(AllOfItem allOfItem) { Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null"); + if (Objects.nonNull(allOfItem.getAllOf())) { + return allOfItem.getAllOf().stream().flatMap(this::processItem); + } + String ref = allOfItem.getRef(); return Objects.isNull(ref) ? @@ -85,7 +89,15 @@ public class PropertiesProcessor { new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef, "Unknown definition:" + definitionSubRef)); - return definition.getProperties().entrySet().stream().flatMap(this::processPropertyEntry); + if (Objects.nonNull(definition.getAllOf())) { + return definition.getAllOf().stream().flatMap(this::processItem); + } + + return processProperties(definition.getProperties()); + } + + public Stream> processProperties(Map properties){ + return properties.entrySet().stream().flatMap(this::processPropertyEntry); } private Stream> processPropertyEntry(Map.Entry entry) { @@ -106,6 +118,13 @@ public class PropertiesProcessor { return Stream.empty(); } + if (Objects.nonNull(entry.getValue().getAllOf())) { + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() + , log, new SchemaConverterPropertiesConfig()); + + return entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem); + } + if (Objects.nonNull(entry.getValue().getProperties())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() , log, new SchemaConverterPropertiesConfig()); @@ -163,7 +182,7 @@ public class PropertiesProcessor { private Supplier getFromFormat(Supplier formatSupplier){ return () -> { - String format = formatSupplier.get();; + String format = formatSupplier.get(); return Objects.nonNull(format) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(format, format) : null; }; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java index d35c8d25..1dc63193 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java @@ -101,6 +101,11 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat { storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef()) .collect(Collectors.toList())); } + + if (schemaData.getProperties() != null) { + storageSchemaItems.addAll(propertiesProcessor.processProperties(schemaData.getProperties()) + .collect(Collectors.toList())); + } } } else { log.warning("Schema doesn't have properties, kind:" + kind); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java index 9499e563..bef9a14b 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java @@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.List; import java.util.Map; @Data @@ -25,4 +26,5 @@ public class AllOfItem { private String ref; private String type; private Map properties; + private List allOf; } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java index f1b820f3..be8292a2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java @@ -16,9 +16,11 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import lombok.Data; +import java.util.List; import java.util.Map; @Data public class Definition { private Map properties; + private List allOf; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java index 706ee8b1..6c09c011 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java @@ -18,10 +18,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import java.util.List; +import java.util.Map; @Data public class PropertiesData { private List allOf; @JsonProperty("$ref") private String ref; + private Map properties; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index 0710bede..3053e5d2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.List; import java.util.Map; @Data @@ -28,4 +29,5 @@ public class TypeProperty { private String ref; private Items items; private Map properties; + private List allOf; } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java index f43d6261..c9009287 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java @@ -39,6 +39,8 @@ import static org.junit.Assert.fail; @SpringBootTest public class SchemaToStorageFormatImplTest { + private static final String KIND = "KIND_VAL"; + private ObjectMapper objectMapper = Jackson2ObjectMapperBuilder.json().build(); private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class); @@ -52,19 +54,24 @@ public class SchemaToStorageFormatImplTest { testSingleFile("/converter/basic/schema.json", "osdu:osdu:Wellbore:1.0.0"); } + @Test + public void rootProperties() { + testSingleFile("/converter/root-properties/schema.json", KIND); + } + @Test public void integrationTestSchema1() { - testSingleFile("/converter/integration-tests/index_records_1.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_1.schema", KIND); } @Test public void integrationTestSchema2() { - testSingleFile("/converter/integration-tests/index_records_2.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_2.schema", KIND); } @Test public void integrationTestSchema3() { - testSingleFile("/converter/integration-tests/index_records_3.schema", "KIND_VAL"); + testSingleFile("/converter/integration-tests/index_records_3.schema", KIND); } @Test @@ -72,6 +79,21 @@ public class SchemaToStorageFormatImplTest { testSingleFile("/converter/wks/slb_wke_wellbore.json", "slb:wks:wellbore:1.0.6"); } + @Test + public void allOfInsideAllOf() { + testSingleFile("/converter/tags/allOf/allOf-inside-allOf.json", KIND); + } + + @Test + public void allOfInsideProperty() { + testSingleFile("/converter/tags/allOf/allOf-inside-property.json", KIND); + } + + @Test + public void allOfInDefinitions() { + testSingleFile("/converter/tags/allOf/indefinitions.json", KIND); + } + @Test public void folderPassed() throws URISyntaxException, IOException { @@ -85,8 +107,9 @@ public class SchemaToStorageFormatImplTest { private void testSingleFile(String filename, String kind) { String json = getSchemaFromSchemaService(filename); - Map expected = getStorageSchema( filename + ".res"); + Map converted = schemaToStorageFormatImpl.convertToMap(json, kind); + Map expected = getStorageSchema( filename + ".res"); compareSchemas(expected, converted, filename); } diff --git a/indexer-core/src/test/resources/converter/root-properties/schema.json b/indexer-core/src/test/resources/converter/root-properties/schema.json new file mode 100644 index 00000000..55fe1a30 --- /dev/null +++ b/indexer-core/src/test/resources/converter/root-properties/schema.json @@ -0,0 +1,12 @@ +{ + "properties": { + "data": { + "properties": { + "FacilityName": { + "type": "string" + } + }, + "type": "object" + } + } +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/root-properties/schema.json.res b/indexer-core/src/test/resources/converter/root-properties/schema.json.res new file mode 100644 index 00000000..cd12f24c --- /dev/null +++ b/indexer-core/src/test/resources/converter/root-properties/schema.json.res @@ -0,0 +1,9 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "FacilityName" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json new file mode 100644 index 00000000..8ba3af25 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json @@ -0,0 +1,56 @@ +{ + "definitions": { + "wellboreData1": { + "properties": { + "prop1": { + "type": "string" + } + } + }, + "wellboreData2": { + "properties": { + "prop2": { + "type": "string" + } + } + }, + "wellboreData3": { + "properties": { + "prop3": { + "type": "string" + } + } + }, + "wellboreData4": { + "properties": { + "prop4": { + "type": "string" + } + } + } + }, + "properties": { + "data": { + "allOf": [ + { + "allOf": [ + { + "$ref": "#/definitions/wellboreData1" + }, + { + "$ref": "#/definitions/wellboreData2" + } + ] + }, + { + "$ref": "#/definitions/wellboreData3" + }, + { + "$ref": "#/definitions/wellboreData4" + } + ] + } + } +} + + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res new file mode 100644 index 00000000..22a8e6a8 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-allOf.json.res @@ -0,0 +1,21 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "prop1" + }, + { + "kind": "string", + "path": "prop2" + }, + { + "kind": "string", + "path": "prop3" + }, + { + "kind": "string", + "path": "prop4" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json new file mode 100644 index 00000000..1f2d761d --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json @@ -0,0 +1,33 @@ +{ + "definitions": { + "def1": { + "properties": { + "prop1": { + "type": "string" + } + } + } + }, + "properties": { + "data": { + "properties": { + "FacilityName": { + "allOf": [ + { + "$ref": "#/definitions/def1" + }, + { + "properties": { + "val" : { + "type": "string" } + } + } + ] + } + }, + "type": "object" + } + } +} + + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res new file mode 100644 index 00000000..147f31d3 --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/allOf-inside-property.json.res @@ -0,0 +1,14 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "FacilityName.prop1" + }, + { + "kind": "string", + "path": "FacilityName.val" + } + + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json new file mode 100644 index 00000000..0064dd0a --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json @@ -0,0 +1,29 @@ +{ + "definitions": { + "wellboreData1": { + "properties": { + "prop1": { + "type": "string" + } + } + }, + "wellboreData2": { + "allOf": [ + { + "$ref": "#/definitions/wellboreData1" + } + ] + } + }, + "properties": { + "data": { + "type": "object", + "properties": { + "Field": { + "$ref": "#/definitions/wellboreData2" + } + } + } + } +} + diff --git a/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res new file mode 100644 index 00000000..e17ce90d --- /dev/null +++ b/indexer-core/src/test/resources/converter/tags/allOf/indefinitions.json.res @@ -0,0 +1,9 @@ +{ + "kind": "KIND_VAL", + "schema": [ + { + "kind": "string", + "path": "Field.prop1" + } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index eef38efd..a85a2d46 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ org.projectlombok lombok - 1.18.8 + 1.18.16 provided -- GitLab From 0776707f8003866f37e02d1c74ff75990392d64a Mon Sep 17 00:00:00 2001 From: Aliaksandr Ramanovich1 Date: Wed, 23 Dec 2020 06:12:02 +0000 Subject: [PATCH 09/11] Parametrize cloudbuild.yml --- provider/indexer-gcp/cloudbuild/cloudbuild.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml index a0839623..601ba2e8 100644 --- a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml +++ b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml @@ -17,15 +17,14 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: [ - 'build', - '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', - '--build-arg', 'PORT=${_PORT}', - '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}', - '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest', - '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', - '.' + 'build', + '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', + '--build-arg', 'PORT=${_PORT}', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest', + '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', + '.' ] images: - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}' - -- GitLab From 6fe3581f144204ce65c4609b31002b9a40e19f58 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Tue, 29 Dec 2020 08:30:46 +0000 Subject: [PATCH 10/11] Update cloudbuild.yaml --- provider/indexer-gcp/cloudbuild/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml index 601ba2e8..1d0c9ba8 100644 --- a/provider/indexer-gcp/cloudbuild/cloudbuild.yaml +++ b/provider/indexer-gcp/cloudbuild/cloudbuild.yaml @@ -28,3 +28,4 @@ steps: images: - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}' + -- GitLab From 0e834a49d2b19b8af5625653ed5a2a94614026d9 Mon Sep 17 00:00:00 2001 From: neelesh thakur Date: Tue, 19 Jan 2021 15:21:47 -0500 Subject: [PATCH 11/11] Update NOTICE --- NOTICE | 3 --- 1 file changed, 3 deletions(-) diff --git a/NOTICE b/NOTICE index 84fce0ab..878e5e17 100644 --- a/NOTICE +++ b/NOTICE @@ -815,7 +815,6 @@ The following software have components provided under the terms of this license: - OSGi resource locator (from ) - Project Lombok (from https://projectlombok.org) -- Project Lombok (from https://projectlombok.org) - javax.ws.rs-api (from http://jax-rs-spec.java.net) ======================================================================== @@ -904,7 +903,6 @@ The following software have components provided under the terms of this license: - Plexus :: Default Container (from ) - Plexus Default Interactivity Handler (from ) - Project Lombok (from https://projectlombok.org) -- Project Lombok (from https://projectlombok.org) - SLF4J API Module (from http://www.slf4j.org) - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) @@ -1006,7 +1004,6 @@ The following software have components provided under the terms of this license: - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Project Lombok (from https://projectlombok.org) -- Project Lombok (from https://projectlombok.org) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) - Spring Web (from https://github.com/spring-projects/spring-framework) -- GitLab