From b4d3d0d7bc057b9b7406360abb31ac7c2aff7a85 Mon Sep 17 00:00:00 2001
From: Heba <heelayot@microsoft.com>
Date: Fri, 13 Sep 2019 23:38:28 +0000
Subject: [PATCH] Merged PR 159: Fix unit tests
-Fix test path for Gcp
-Add MockMaker property file for Mockito
Related work items: #159
---
.gitignore | 36 ++
.idea/$PRODUCT_WORKSPACE_FILE$ | 19 +
.idea/.gitignore | 2 +
.idea/sbt.xml | 7 +
dps-indexer-queue/pom.xml | 15 -
dps-indexer/pom.xml | 15 -
indexer-service-azure/pom.xml | 76 +++
.../main/java/IndexerAzureApplication.java | 9 +
.../util/di/EntitlementsClientFactory.java | 21 +
.../util/di/EntitlementsFactoryAzure.java | 12 +
.../util/di/EntitlementsServiceAzure.java | 87 +++
.../azure/util/di/TenantFactoryImpl.java | 45 ++
.../indexer/azure/util/kms/KmsClientImpl.java | 17 +
.../ElasticRepositoryCosmosDB.java | 12 +
.../persistence/SchemaRepositoryImpl.java | 63 ++
.../azure/util/util/RequestInfoImpl.java | 75 +++
.../util/ServiceAccountJwtClientImpl.java | 12 +
indexer-service-gcp/pom.xml | 134 ++++
.../src/main/java/IndexerGcpApplication.java | 11 +
.../gcp/cache/DatastoreCredentialCache.java | 27 +
.../opendes/indexer/gcp/kms/KmsClient.java | 96 +++
.../indexer/gcp/model/AppEngineHeaders.java | 28 +
.../gcp/persistence/DatastoreCredential.java | 109 ++++
.../gcp/persistence/DatastoreFactory.java | 65 ++
.../ElasticRepositoryDatastore.java | 81 +++
.../indexer/gcp/publish/PublisherImpl.java | 130 ++++
.../indexer/gcp/util/HeadersInfoGcpImpl.java | 115 ++++
.../indexer/gcp/util/RequestInfoImpl.java | 85 +++
.../util/ServiceAccountJwtGcpClientImpl.java | 181 ++++++
.../indexer/gcp/util/TraceIdExtractor.java | 63 ++
.../gcp/util/HeadersInfoGcpImplTest.java | 198 ++++++
.../ServiceAccountJwtGcpClientImplTest.java | 178 ++++++
.../gcp/util/TraceIdExtractorTest.java | 71 +++
.../org.mockito.plugins.MockMaker | 1 +
indexer-service-root/maven/settings.xml | 14 +
indexer-service-root/mvnw | 286 +++++++++
indexer-service-root/mvnw.cmd | 161 +++++
indexer-service-root/pom.xml | 145 +++++
.../java/org/opendes/indexer/SwaggerDoc.java | 148 +++++
.../opendes/indexer/api/HealthCheckApi.java | 40 ++
.../opendes/indexer/api/RecordIndexerApi.java | 70 +++
.../org/opendes}/indexer/api/ReindexApi.java | 59 +-
.../opendes/indexer/cache/SchemaCache.java | 25 +
.../opendes/indexer/logging/AuditEvents.java | 221 +++++++
.../opendes/indexer/logging/AuditLogger.java | 100 +++
.../opendes/indexer/logging/ServiceLogId.java | 35 ++
.../indexer/middleware/IndexerFilter.java | 144 +++++
.../RedirectHttpRequestsHandler.java | 43 ++
.../indexer/model/ConversionStatus.java | 28 +
.../opendes/indexer/model/ElasticType.java | 68 ++
.../opendes/indexer/model/IndexProgress.java | 28 +
.../opendes/indexer/model/IndexSchema.java | 47 ++
.../opendes/indexer/model/IndexingStatus.java | 38 ++
.../java/org/opendes/indexer/model/Legal.java | 34 +
.../indexer/model/MultiFieldIndexRequest.java | 40 ++
.../opendes/indexer/model/OperationType.java | 57 ++
.../opendes/indexer/model/PublishMessage.java | 5 +
.../opendes/indexer/model/RecordAncestry.java | 23 +
.../org/opendes/indexer/model/RecordIds.java | 30 +
.../indexer/model/RecordIndexerPayload.java | 57 ++
.../indexer/model/RecordQueryResponse.java | 29 +
.../indexer/model/RecordReindexRequest.java | 36 ++
.../opendes/indexer/model/RecordStatus.java | 43 ++
.../org/opendes/indexer/model/Records.java | 60 ++
.../org/opendes/indexer/model/Schema.java | 39 ++
.../org/opendes/indexer/model/StorageAcl.java | 42 ++
.../opendes/indexer/model/StorageType.java | 52 ++
.../opendes/indexer/publish/IPublisher.java | 9 +
.../service/AttributeParsingServiceImpl.java | 194 ++++++
.../opendes/indexer/service/CronService.java | 24 +
.../indexer/service/CronServiceImpl.java | 102 +++
.../service/GeometryConversionService.java | 127 ++++
.../service/IAttributeParsingService.java | 29 +
.../indexer/service/IndexCopyService.java | 26 +
.../indexer/service/IndexCopyServiceImpl.java | 215 +++++++
.../indexer/service/IndexSchemaService.java | 29 +
.../service/IndexSchemaServiceImpl.java | 207 +++++++
.../service/IndexerMappingService.java | 32 +
.../service/IndexerMappingServiceImpl.java | 330 ++++++++++
.../indexer/service/IndexerService.java | 27 +
.../indexer/service/IndexerServiceImpl.java | 585 ++++++++++++++++++
.../indexer/service/ReindexService.java | 23 +
.../indexer/service/ReindexServiceImpl.java | 88 +++
.../indexer/service/StorageService.java | 33 +
.../indexer/service/StorageServiceImpl.java | 136 ++++
.../indexer/swagger/HomeController.java | 13 +
.../swagger/SwaggerDocumentationConfig.java | 76 +++
.../opendes/indexer/util/IRequestInfo.java | 22 +
.../indexer/util/IndexerQueueTaskBuilder.java | 66 ++
.../org/opendes/indexer/util/JobStatus.java | 132 ++++
.../org/opendes/indexer/util/RecordInfo.java | 98 +++
.../org/opendes/indexer/util/TypeMapper.java | 113 ++++
.../indexer/util/parser/DateTimeParser.java | 84 +++
.../indexer/util/parser/NumberParser.java | 147 +++++
.../indexer/api/HealthCheckApiTest.java | 46 ++
.../indexer/api/RecordIndexerApiTest.java | 121 ++++
.../opendes/indexer/api/ReindexApiTest.java | 74 +++
.../indexer/logging/AuditEventsTest.java | 230 +++++++
.../indexer/logging/AuditLoggerTest.java | 210 +++++++
.../indexer/logging/JaxRsDpsLogTest.java | 114 ++++
.../indexer/middleware/IndexerFilterTest.java | 128 ++++
.../RedirectHttpRequestsHandlerTest.java | 104 ++++
.../indexer/model/ElasticTypeTest.java | 51 ++
.../indexer/model/IndexingStatusTest.java | 44 ++
.../model/RecordChagedMessagesTest.java | 91 +++
.../indexer/model/StorageTypeTest.java | 40 ++
.../AttributeParsingServiceImplTest.java | 219 +++++++
.../indexer/service/CronServiceImplTest.java | 141 +++++
.../GeometryConversionServiceTest.java | 124 ++++
.../service/IndexCopyServiceImplTest.java | 184 ++++++
.../service/IndexerMappingServiceTest.java | 324 ++++++++++
.../service/IndexerSchemaServiceTest.java | 325 ++++++++++
.../indexer/service/IndexerServiceTest.java | 337 ++++++++++
.../indexer/service/ReindexServiceTest.java | 144 +++++
.../indexer/service/StorageServiceTest.java | 210 +++++++
.../opendes/indexer/util/JobStatusTest.java | 239 +++++++
.../opendes/indexer/util/RecordInfoTest.java | 102 +++
.../util/parser/DateTimeParserTest.java | 92 +++
.../indexer/util/parser/NumberParserTest.java | 205 ++++++
.../org.mockito.plugins.MockMaker | 1 +
pom.xml | 164 ++++-
121 files changed, 11307 insertions(+), 67 deletions(-)
create mode 100644 .gitignore
create mode 100644 .idea/$PRODUCT_WORKSPACE_FILE$
create mode 100644 .idea/.gitignore
create mode 100644 .idea/sbt.xml
delete mode 100644 dps-indexer-queue/pom.xml
delete mode 100644 dps-indexer/pom.xml
create mode 100644 indexer-service-azure/pom.xml
create mode 100644 indexer-service-azure/src/main/java/IndexerAzureApplication.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsClientFactory.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsFactoryAzure.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsServiceAzure.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/TenantFactoryImpl.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/kms/KmsClientImpl.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/ElasticRepositoryCosmosDB.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/SchemaRepositoryImpl.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/RequestInfoImpl.java
create mode 100644 indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/ServiceAccountJwtClientImpl.java
create mode 100644 indexer-service-gcp/pom.xml
create mode 100644 indexer-service-gcp/src/main/java/IndexerGcpApplication.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/cache/DatastoreCredentialCache.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/kms/KmsClient.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/model/AppEngineHeaders.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreCredential.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreFactory.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/ElasticRepositoryDatastore.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/publish/PublisherImpl.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImpl.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/RequestInfoImpl.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImpl.java
create mode 100644 indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/TraceIdExtractor.java
create mode 100644 indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImplTest.java
create mode 100644 indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImplTest.java
create mode 100644 indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/TraceIdExtractorTest.java
create mode 100644 indexer-service-gcp/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
create mode 100644 indexer-service-root/maven/settings.xml
create mode 100644 indexer-service-root/mvnw
create mode 100644 indexer-service-root/mvnw.cmd
create mode 100644 indexer-service-root/pom.xml
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/SwaggerDoc.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/api/HealthCheckApi.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/api/RecordIndexerApi.java
rename {dps-indexer/src/main/java/com/slb => indexer-service-root/src/main/java/org/opendes}/indexer/api/ReindexApi.java (60%)
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/cache/SchemaCache.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditEvents.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditLogger.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/logging/ServiceLogId.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/middleware/IndexerFilter.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandler.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/ConversionStatus.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/ElasticType.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/IndexProgress.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/IndexSchema.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/IndexingStatus.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/Legal.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/MultiFieldIndexRequest.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/OperationType.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/PublishMessage.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordAncestry.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIds.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIndexerPayload.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordQueryResponse.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordReindexRequest.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/RecordStatus.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/Records.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/Schema.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/StorageAcl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/model/StorageType.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/publish/IPublisher.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/AttributeParsingServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/CronService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/CronServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/GeometryConversionService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IAttributeParsingService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/StorageService.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/service/StorageServiceImpl.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/swagger/HomeController.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/swagger/SwaggerDocumentationConfig.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/IRequestInfo.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/IndexerQueueTaskBuilder.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/JobStatus.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/RecordInfo.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/TypeMapper.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/parser/DateTimeParser.java
create mode 100644 indexer-service-root/src/main/java/org/opendes/indexer/util/parser/NumberParser.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/api/HealthCheckApiTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/api/RecordIndexerApiTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/api/ReindexApiTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditEventsTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditLoggerTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/logging/JaxRsDpsLogTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/middleware/IndexerFilterTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandlerTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/model/ElasticTypeTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/model/IndexingStatusTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/model/RecordChagedMessagesTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/model/StorageTypeTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/AttributeParsingServiceImplTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/CronServiceImplTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/GeometryConversionServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/IndexCopyServiceImplTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerMappingServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerSchemaServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/ReindexServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/service/StorageServiceTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/util/JobStatusTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/util/RecordInfoTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/util/parser/DateTimeParserTest.java
create mode 100644 indexer-service-root/src/test/java/org/opendes/indexer/util/parser/NumberParserTest.java
create mode 100644 indexer-service-root/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..76d963dc1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,36 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**
+!**/src/test/**
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+
+### VS Code ###
+.vscode/
+
+### Other ###
+.mvn
+target/*
+*/target
\ No newline at end of file
diff --git a/.idea/$PRODUCT_WORKSPACE_FILE$ b/.idea/$PRODUCT_WORKSPACE_FILE$
new file mode 100644
index 000000000..3733e0d36
--- /dev/null
+++ b/.idea/$PRODUCT_WORKSPACE_FILE$
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+ <component name="masterDetails">
+ <states>
+ <state key="ProjectJDKs.UI">
+ <settings>
+ <last-edited>1.8</last-edited>
+ <splitter-proportions>
+ <option name="proportions">
+ <list>
+ <option value="0.2" />
+ </list>
+ </option>
+ </splitter-proportions>
+ </settings>
+ </state>
+ </states>
+ </component>
+</project>
\ No newline at end of file
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 000000000..5c98b4288
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,2 @@
+# Default ignored files
+/workspace.xml
\ No newline at end of file
diff --git a/.idea/sbt.xml b/.idea/sbt.xml
new file mode 100644
index 000000000..5453d20eb
--- /dev/null
+++ b/.idea/sbt.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+ <component name="ScalaSbtSettings">
+ <option name="customVMEnabled" value="true" />
+ <option name="customVMPath" value="C:/java-1.8.0-openjdk-1.8.0.212-3.b04.redhat.windows.x86_64" />
+ </component>
+</project>
\ No newline at end of file
diff --git a/dps-indexer-queue/pom.xml b/dps-indexer-queue/pom.xml
deleted file mode 100644
index 5670b8ddd..000000000
--- a/dps-indexer-queue/pom.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>dps-indexer-boot-mvn</artifactId>
- <groupId>com.slb</groupId>
- <version>1.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>dps-indexer-queue</artifactId>
-
-
-</project>
\ No newline at end of file
diff --git a/dps-indexer/pom.xml b/dps-indexer/pom.xml
deleted file mode 100644
index 445d1efd2..000000000
--- a/dps-indexer/pom.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>dps-indexer-boot-mvn</artifactId>
- <groupId>com.slb</groupId>
- <version>1.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>dps-indexer</artifactId>
-
-
-</project>
\ No newline at end of file
diff --git a/indexer-service-azure/pom.xml b/indexer-service-azure/pom.xml
new file mode 100644
index 000000000..53660ea24
--- /dev/null
+++ b/indexer-service-azure/pom.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>indexer-service-azure</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>indexer-service-azure</name>
+ <description>Indexer Service Azure</description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <azure.version>2.1.7</azure.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service-root</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-active-directory-spring-boot-starter</artifactId>
+ <version>${azure.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-cosmosdb-spring-boot-starter</artifactId>
+ <version>${azure.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId >
+ <artifactId>azure-storage-spring-boot-starter</artifactId>
+ <version>${azure.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-servicebus-spring-boot-starter</artifactId>
+ <version>${azure.version}</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opendes.indexer.IndexerAzureApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
\ No newline at end of file
diff --git a/indexer-service-azure/src/main/java/IndexerAzureApplication.java b/indexer-service-azure/src/main/java/IndexerAzureApplication.java
new file mode 100644
index 000000000..4e2e80363
--- /dev/null
+++ b/indexer-service-azure/src/main/java/IndexerAzureApplication.java
@@ -0,0 +1,9 @@
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class IndexerAzureApplication {
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerAzureApplication.class, args);
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsClientFactory.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsClientFactory.java
new file mode 100644
index 000000000..9a3557626
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsClientFactory.java
@@ -0,0 +1,21 @@
+package org.opendes.indexer.azure.util.di;
+
+import org.opendes.client.api.entitlements.IEntitlementsFactory;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Component("AzureEntitlementsClientFactory")
+@Primary
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+ @Override
+ protected IEntitlementsFactory createInstance() throws Exception {
+ return new EntitlementsFactoryAzure();
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return IEntitlementsFactory.class;
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsFactoryAzure.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsFactoryAzure.java
new file mode 100644
index 000000000..ee5fd8280
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsFactoryAzure.java
@@ -0,0 +1,12 @@
+package org.opendes.indexer.azure.util.di;
+
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.api.entitlements.IEntitlementsFactory;
+import org.opendes.client.api.entitlements.IEntitlementsService;
+
+public class EntitlementsFactoryAzure implements IEntitlementsFactory {
+ @Override
+ public IEntitlementsService create(DpsHeaders headers) {
+ return new EntitlementsServiceAzure(headers);
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsServiceAzure.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsServiceAzure.java
new file mode 100644
index 000000000..eb20161ad
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/EntitlementsServiceAzure.java
@@ -0,0 +1,87 @@
+package org.opendes.indexer.azure.util.di;
+
+import com.microsoft.azure.spring.autoconfigure.aad.UserPrincipal;
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.api.entitlements.EntitlementsException;
+import org.opendes.client.api.entitlements.IEntitlementsService;
+import org.opendes.client.api.entitlements.models.*;
+import org.opendes.client.httpclient.HttpResponse;
+import org.opendes.core.model.SearchServiceRole;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class EntitlementsServiceAzure implements IEntitlementsService {
+ DpsHeaders headers;
+
+ public EntitlementsServiceAzure(DpsHeaders headers){
+ this.headers = headers;
+ }
+
+ @Override
+ public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public Groups getGroups() throws EntitlementsException {
+ final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+ final UserPrincipal current = (UserPrincipal) auth.getPrincipal();
+ String email = String.valueOf(current.getUpn());
+
+ List<GroupInfo> giList = new ArrayList();
+ Collection<? extends GrantedAuthority> authorities = auth.getAuthorities();
+ for(GrantedAuthority authority : authorities)
+ {
+ GroupInfo gi = new GroupInfo();
+ String role = authority.getAuthority();
+ if (role.startsWith(SearchServiceRole.PREFIX)){
+ role = role.substring(SearchServiceRole.PREFIX.length());
+ }
+ gi.setName(role);
+ gi.setEmail(email);
+ giList.add(gi);
+ }
+ if (giList.size() > 0)
+ {
+ Groups groups = new Groups();
+ groups.setGroups(giList);
+ groups.setDesId(email);
+ return groups;
+ }
+
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+ throw new EntitlementsException("no authorities found", response);
+ }
+
+ @Override
+ public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public void deleteMember(String s, String s1) throws EntitlementsException {
+
+ }
+
+ @Override
+ public Groups authorizeAny(String... strings) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public void authenticate() throws EntitlementsException {
+
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/TenantFactoryImpl.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/TenantFactoryImpl.java
new file mode 100644
index 000000000..6ecf8c008
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/di/TenantFactoryImpl.java
@@ -0,0 +1,45 @@
+package org.opendes.indexer.azure.util.di;
+
+import org.opendes.client.cache.ICache;
+import org.opendes.client.multitenancy.ITenantFactory;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+ public static final String DefaultTenantName = "common";
+ private List<TenantInfo> tenants;
+
+ public TenantFactoryImpl()
+ {
+ TenantInfo ti = new TenantInfo();
+ ti.setName(DefaultTenantName);
+ this.tenants = new ArrayList<>();
+ this.tenants.add(ti);
+ }
+
+ public boolean exists(String tenantName)
+ {
+ return true;
+ }
+
+ public TenantInfo getTenantInfo(String tenantName) {
+ // we are not checking tenantName yet, we have only 1 tenant
+ return this.tenants.get(0);
+ }
+
+ public Collection<TenantInfo> listTenantInfo() {
+ return this.tenants;
+ }
+
+ public <V> ICache<String, V> createCache(String tenantName, String host, int port, int expireTimeSeconds, Class<V> classOfV)
+ {
+ return null;
+ }
+
+ public void flushCache() {}
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/kms/KmsClientImpl.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/kms/KmsClientImpl.java
new file mode 100644
index 000000000..1743a82c7
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/kms/KmsClientImpl.java
@@ -0,0 +1,17 @@
+package org.opendes.indexer.azure.util.kms;
+
+import org.opendes.core.kms.IKmsClient;
+
+import java.io.IOException;
+
+public class KmsClientImpl implements IKmsClient {
+ @Override
+ public String encryptString(String textToBeEncrypted) throws IOException {
+ return textToBeEncrypted;
+ }
+
+ @Override
+ public String decryptString(String textToBeDecrypted) throws IOException {
+ return textToBeDecrypted;
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/ElasticRepositoryCosmosDB.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/ElasticRepositoryCosmosDB.java
new file mode 100644
index 000000000..e1b79ca8b
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/ElasticRepositoryCosmosDB.java
@@ -0,0 +1,12 @@
+package org.opendes.indexer.azure.util.persistence;
+
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.ClusterSettings;
+import org.opendes.core.persistence.ElasticRepository;
+
+public class ElasticRepositoryCosmosDB implements ElasticRepository {
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+ return null;
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/SchemaRepositoryImpl.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/SchemaRepositoryImpl.java
new file mode 100644
index 000000000..7a3e79537
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/persistence/SchemaRepositoryImpl.java
@@ -0,0 +1,63 @@
+package org.opendes.indexer.azure.util.persistence;
+
+import com.microsoft.azure.spring.data.cosmosdb.core.mapping.Document;
+import com.microsoft.azure.spring.data.cosmosdb.core.mapping.PartitionKey;
+import com.microsoft.azure.spring.data.cosmosdb.repository.DocumentDbRepository;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opendes.core.model.Schema;
+import org.opendes.core.model.SchemaItem;
+import org.opendes.core.service.ISchemaRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.annotation.Id;
+import org.springframework.stereotype.Repository;
+
+import java.util.Map;
+import java.util.Optional;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Document(collection = "StorageSchema") //collection name
+class SchemaDoc {
+
+ @PartitionKey
+ @Id
+ private String kind;
+ private Map<String,Object> extension;
+ private String user;
+ private SchemaItem[] schemaItems;
+}
+
+interface CosmosDB extends DocumentDbRepository<SchemaDoc, String> {}
+
+@Repository
+public class SchemaRepositoryImpl implements ISchemaRepository {
+
+ @Autowired
+ private CosmosDB db;
+
+ @Override
+ public void add(Schema schema, String user) {
+ SchemaDoc sd = new SchemaDoc();
+ sd.setKind(schema.getKind());
+ sd.setExtension(schema.getExt());
+ sd.setUser(user);
+ sd.setSchemaItems(schema.getSchema());
+ db.save(sd);
+ }
+
+ @Override
+ public Schema get(String kind) {
+ Optional<SchemaDoc> sd = db.findById(kind);
+ if (!sd.isPresent())
+ return null;
+
+ Schema newSchema = new Schema();
+ newSchema.setKind(kind);
+ newSchema.setSchema(sd.get().getSchemaItems());
+ newSchema.setExt(sd.get().getExtension());
+ return newSchema;
+ }
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/RequestInfoImpl.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/RequestInfoImpl.java
new file mode 100644
index 000000000..c70f59b52
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/RequestInfoImpl.java
@@ -0,0 +1,75 @@
+package org.opendes.indexer.azure.util.util;
+
+import com.google.common.base.Strings;
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.HeadersInfo;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+
+import static org.opendes.client.api.DpsHeaders.AUTHORIZATION;
+
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Autowired
+ private HeadersInfo headersInfo;
+ @Autowired
+ private ServiceAccountJwtClientImpl serviceAccountJwtClient;
+
+ private static final String expectedCronHeaderValue = "true";
+
+ @Autowired
+ private TenantInfo tenantInfo;
+
+ @Override
+ public DpsHeaders getHeaders() {
+ return this.headersInfo.getHeaders();
+ }
+
+ @Override
+ public String getPartitionId() {
+ return this.headersInfo.getPartitionId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return this.headersInfo.getHeadersMap();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ DpsHeaders output = this.headersInfo.getHeaders();
+ output.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return output;
+ }
+
+ @Override
+ public boolean isCronRequest() { return false; }
+
+ @Override
+ public boolean isTaskQueueRequest() { return false; }
+
+ private String checkOrGetAuthorizationHeader() {
+ if (Config.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.headersInfo.getHeaders().getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
+ }
+ return authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+
+}
diff --git a/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/ServiceAccountJwtClientImpl.java b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/ServiceAccountJwtClientImpl.java
new file mode 100644
index 000000000..440b05b58
--- /dev/null
+++ b/indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/util/ServiceAccountJwtClientImpl.java
@@ -0,0 +1,12 @@
+package org.opendes.indexer.azure.util.util;
+
+import org.opendes.core.util.IServiceAccountJwtClient;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+ @Override
+ public String getIdToken(String tenantName){
+ return "common";
+ }
+}
diff --git a/indexer-service-gcp/pom.xml b/indexer-service-gcp/pom.xml
new file mode 100644
index 000000000..1ac5b18cf
--- /dev/null
+++ b/indexer-service-gcp/pom.xml
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>indexer-service-gcp</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>indexer-service-gcp</name>
+ <description>Indexer Service GCP</description>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service-root</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.os</groupId>
+ <artifactId>client-lib-gcp</artifactId>
+ <version>0.0.8</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.os</groupId>
+ <artifactId>client-lib</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-datastore</artifactId>
+ <version>1.72.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-logging</artifactId>
+ <version>1.72.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-storage</artifactId>
+ <version>v1-rev150-1.25.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-jdk5</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-cloudkms</artifactId>
+ <version>v1-rev81-1.25.0</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-pubsub -->
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-pubsub</artifactId>
+ <version>1.60.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.appengine.tools</groupId>
+ <artifactId>appengine-gcs-client</artifactId>
+ <version>0.8</version>
+ </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.mockito</groupId>-->
+<!-- <artifactId>mockito-core</artifactId>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>junit</groupId>-->
+<!-- <artifactId>junit</artifactId>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.powermock</groupId>-->
+<!-- <artifactId>powermock-core</artifactId>-->
+<!-- <version>2.0.2</version>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.powermock</groupId>-->
+<!-- <artifactId>powermock-api-mockito2</artifactId>-->
+<!-- <version>2.0.2</version>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.springframework</groupId>-->
+<!-- <artifactId>spring-test</artifactId>-->
+<!-- <version>5.1.9.RELEASE</version>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.springframework</groupId>-->
+<!-- <artifactId>spring-test</artifactId>-->
+<!-- <version>5.1.9.RELEASE</version>-->
+<!-- <scope>test</scope>-->
+<!-- </dependency>-->
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opendes.indexer.IndexerGcpApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/IndexerGcpApplication.java b/indexer-service-gcp/src/main/java/IndexerGcpApplication.java
new file mode 100644
index 000000000..5ad552ea7
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/IndexerGcpApplication.java
@@ -0,0 +1,11 @@
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class IndexerGcpApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerGcpApplication.class, args);
+ }
+
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/cache/DatastoreCredentialCache.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/cache/DatastoreCredentialCache.java
new file mode 100644
index 000000000..49f1b4399
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/cache/DatastoreCredentialCache.java
@@ -0,0 +1,27 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.cache;
+
+import com.google.auth.oauth2.AccessToken;
+import org.opendes.client.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+
+public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
+
+ // Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration
+ public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) {
+ super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class);
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/kms/KmsClient.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/kms/KmsClient.java
new file mode 100644
index 000000000..a57b23676
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/kms/KmsClient.java
@@ -0,0 +1,96 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.kms;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.cloudkms.v1.CloudKMS;
+import com.google.api.services.cloudkms.v1.CloudKMSScopes;
+import com.google.api.services.cloudkms.v1.model.DecryptRequest;
+import com.google.api.services.cloudkms.v1.model.DecryptResponse;
+import com.google.api.services.cloudkms.v1.model.EncryptRequest;
+import com.google.api.services.cloudkms.v1.model.EncryptResponse;
+import org.opendes.core.kms.IKmsClient;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.Preconditions;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class KmsClient implements IKmsClient {
+
+ @Value("${KMS_KEY}")
+ private String KMS_KEY;
+
+ @Value("${KEY_RING}")
+ private String KEY_RING;
+
+ private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s";
+
+ /**
+ * Encrypts the given plaintext using the specified crypto key.
+ * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation
+ */
+ public String encryptString(String textToBeEncrypted) throws IOException {
+ Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null");
+
+ byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8);
+ String resourceName = String.format(KEY_NAME, Config.getGoogleCloudProjectId(), KEY_RING, KMS_KEY);
+ CloudKMS kms = createAuthorizedClient();
+ EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext);
+ EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+ .encrypt(resourceName, request)
+ .execute();
+ return response.getCiphertext();
+ }
+
+ /**
+ * Decrypts the provided ciphertext with the specified crypto key.
+ * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled
+ */
+ public String decryptString(String textToBeDecrypted) throws IOException {
+ Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null");
+
+ CloudKMS kms = createAuthorizedClient();
+ String cryptoKeyName = String.format(KEY_NAME, Config.getGoogleCloudProjectId(), KEY_RING, KMS_KEY);
+ DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted);
+ DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+ .decrypt(cryptoKeyName, request)
+ .execute();
+ return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim();
+ }
+
+ /**
+ * Creates an authorized CloudKMS client service using Application Default Credentials.
+ *
+ * @return an authorized CloudKMS client
+ * @throws IOException if there's an error getting the default credentials.
+ */
+ private CloudKMS createAuthorizedClient() throws IOException {
+ HttpTransport transport = new NetHttpTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ credential = credential.createScoped(CloudKMSScopes.all());
+ }
+ return new CloudKMS.Builder(transport, jsonFactory, credential)
+ .setApplicationName("CloudKMS snippets")
+ .build();
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/model/AppEngineHeaders.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/model/AppEngineHeaders.java
new file mode 100644
index 000000000..9c14a63a5
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/model/AppEngineHeaders.java
@@ -0,0 +1,28 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.model;
+
+public class AppEngineHeaders {
+ public static final String DATA_GROUPS = "X-Data-Groups";
+ public static final String TASK_QUEUE_RETRY_COUNT = "X-AppEngine-TaskExecutionCount";
+ public static final String TASK_QUEUE_NAME = "X-AppEngine-QueueName";
+ public static final String CITY_LAT_LONG = "X-AppEngine-CityLatLong";
+ public static final String COUNTRY = "X-AppEngine-Country";
+ public static final String REGION = "X-AppEngine-Region";
+ public static final String CITY = "X-AppEngine-City";
+ public static final String CLOUD_TRACE_CONTEXT = "X-Cloud-Trace-Context";
+ public static final String TRACE_ID = "X-Trace-Id";
+ public static final String CRON_SERVICE = "X-AppEngine-Cron";
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreCredential.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreCredential.java
new file mode 100644
index 000000000..4a8d7284a
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreCredential.java
@@ -0,0 +1,109 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.persistence;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.gson.JsonObject;
+import org.apache.commons.lang3.time.DateUtils;
+import org.opendes.client.cryptographic.Crc32c;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.indexer.gcp.cache.DatastoreCredentialCache;
+
+import java.util.Date;
+
+public class DatastoreCredential extends GoogleCredentials {
+
+ private static final long serialVersionUID = 8344377091688956815L;
+ private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private Iam iam;
+
+ private final TenantInfo tenant;
+ private final DatastoreCredentialCache cache;
+
+ protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
+ this.tenant = tenant;
+ this.cache = cache;
+ }
+
+ @Override
+ public AccessToken refreshAccessToken() {
+
+ String cacheKey = this.getCacheKey();
+
+ AccessToken accessToken = this.cache.get(cacheKey);
+
+ if (accessToken != null) {
+ return accessToken;
+ }
+
+ try {
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(this.getPayload());
+
+ String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
+
+ SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest);
+
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
+
+ this.cache.put(cacheKey, accessToken);
+
+ return accessToken;
+ } catch (Exception e) {
+ throw new RuntimeException("Error creating datastore credential", e);
+ }
+ }
+
+ private String getPayload() {
+ JsonObject payload = new JsonObject();
+ payload.addProperty("iss", this.tenant.getServiceAccount());
+ payload.addProperty("sub", this.tenant.getServiceAccount());
+ payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
+ payload.addProperty("iat", System.currentTimeMillis() / 1000);
+
+ return payload.toString();
+ }
+
+ protected void setIam(Iam iam) {
+ this.iam = iam;
+ }
+
+ private Iam getIam() throws Exception {
+ if (this.iam == null) {
+
+ Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
+ GoogleCredential.getApplicationDefault()).setApplicationName("Search Service");
+
+ this.iam = builder.build();
+ }
+ return this.iam;
+ }
+
+ private String getCacheKey() {
+ return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreFactory.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreFactory.java
new file mode 100644
index 000000000..5d55a6863
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreFactory.java
@@ -0,0 +1,65 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.persistence;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreOptions;
+import com.google.cloud.http.HttpTransportOptions;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.indexer.gcp.cache.DatastoreCredentialCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.threeten.bp.Duration;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DatastoreFactory {
+
+ @Autowired
+ private DatastoreCredentialCache cache;
+
+ private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
+
+ private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
+ .setMaxAttempts(6)
+ .setInitialRetryDelay(Duration.ofSeconds(10))
+ .setMaxRetryDelay(Duration.ofSeconds(32))
+ .setRetryDelayMultiplier(2.0)
+ .setTotalTimeout(Duration.ofSeconds(50))
+ .setInitialRpcTimeout(Duration.ofSeconds(50))
+ .setRpcTimeoutMultiplier(1.0)
+ .setMaxRpcTimeout(Duration.ofSeconds(50))
+ .build();
+
+ private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
+ .setReadTimeout(30000)
+ .build();
+
+ public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
+ if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
+ Datastore googleDatastore = DatastoreOptions.newBuilder()
+ .setCredentials(new DatastoreCredential(tenantInfo, this.cache))
+ .setRetrySettings(RETRY_SETTINGS)
+ .setTransportOptions(TRANSPORT_OPTIONS)
+ .setNamespace(tenantInfo.getName())
+ .setProjectId(tenantInfo.getProjectId())
+ .build().getService();
+ DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
+ }
+ return DATASTORE_CLIENTS.get(tenantInfo.getName());
+ }
+}
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/ElasticRepositoryDatastore.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/ElasticRepositoryDatastore.java
new file mode 100644
index 000000000..fed47389b
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/ElasticRepositoryDatastore.java
@@ -0,0 +1,81 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.persistence;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.Entity;
+import com.google.cloud.datastore.Key;
+import org.apache.http.HttpStatus;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.ClusterSettings;
+import org.opendes.core.persistence.ElasticRepository;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.Preconditions;
+import org.opendes.indexer.gcp.kms.KmsClient;
+import org.springframework.beans.factory.annotation.Autowired;
+
+
+public class ElasticRepositoryDatastore implements ElasticRepository {
+
+ static final String HOST = "host";
+ static final String PORT = "port";
+ static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration";
+
+ @Autowired
+ private KmsClient kmsClient;
+ @Autowired
+ private DatastoreFactory datastoreFactory;
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+
+ Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo);
+ Key key = googleDatastore.newKeyFactory().setKind(Config.getElasticCredentialsDatastoreKind()).newKey(Config.getElasticCredentialsDatastoreId());
+ Entity datastoreEntity = googleDatastore.get(key);
+
+ if (datastoreEntity == null) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Cluster setting not found", "The requested cluster setting was not found in datastore.", String.format("Cluster setting with key: '%s' does not exist in datastore.", key.getName()));
+ }
+
+ String encryptedHost = null;
+ String encryptedPort = null;
+ String encryptedConfiguration = null;
+
+ try {
+ encryptedHost = datastoreEntity.getString(HOST);
+ encryptedPort = datastoreEntity.getString(PORT);
+ encryptedConfiguration = datastoreEntity.getString(XPACK_RESTCLIENT_CONFIGURATION);
+
+ String host = this.kmsClient.decryptString(encryptedHost);
+ String portString = this.kmsClient.decryptString(encryptedPort);
+ String usernameAndPassword = this.kmsClient.decryptString(encryptedConfiguration);
+
+ Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
+ Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
+ Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
+
+ int port = Integer.parseInt(portString);
+
+ return new ClusterSettings(host, port, usernameAndPassword);
+ } catch (GoogleJsonResponseException e) {
+ String debuggingInfo = String.format("Host: %s | port: %s | configuration: %s", encryptedHost, encryptedPort, encryptedConfiguration);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting decryption error", "An error has occurred decrypting cluster settings.", debuggingInfo, e);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", "An error has occurred fetching cluster settings from the datastore.", e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/publish/PublisherImpl.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/publish/PublisherImpl.java
new file mode 100644
index 000000000..67198fb06
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/publish/PublisherImpl.java
@@ -0,0 +1,130 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.publish;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.pubsub.v1.Publisher;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.reflect.TypeToken;
+import com.google.protobuf.ByteString;
+import com.google.pubsub.v1.ProjectTopicName;
+import com.google.pubsub.v1.PubsubMessage;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.common.Strings;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.gcp.PubSub.PubSubExtensions;
+import org.opendes.client.multitenancy.ITenantFactory;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.opendes.indexer.model.RecordStatus;
+import org.opendes.indexer.publish.IPublisher;
+import org.opendes.indexer.util.JobStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.threeten.bp.Duration;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PublisherImpl implements IPublisher {
+
+ private static final Map<String, Publisher> PUBSUB_CLIENTS = new HashMap<>();
+
+ private static final String TOPIC_ID = "indexing-progress";
+
+
+ @Autowired
+ private ITenantFactory tenantStorageFactory;
+
+ @Autowired
+ private PubSubExtensions pubSubExtensions;
+
+ @Override
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
+
+ if (Config.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL);
+
+ String tenant = headers.getPartitionId();
+ if(Strings.isNullOrEmpty(tenant))
+ tenant = headers.getAccountId();
+
+ Publisher publisher = this.getPublisher(tenant);
+ if (publisher == null) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error occurred creating publisher client.");
+ }
+ PubsubMessage pubsubMessage = getPubsubMessage(headers, indexerBatchStatus);
+
+ pubSubExtensions.publishAndCreateTopicIfNotExist(publisher, pubsubMessage);
+
+ }
+
+ private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
+ .setTotalTimeout(Duration.ofSeconds(30))
+ .setInitialRetryDelay(Duration.ofSeconds(2))
+ .setRetryDelayMultiplier(2)
+ .setMaxRetryDelay(Duration.ofSeconds(5))
+ .setInitialRpcTimeout(Duration.ofSeconds(10))
+ .setRpcTimeoutMultiplier(2)
+ .setMaxRpcTimeout(Duration.ofSeconds(10))
+ .build();
+
+
+ private PubsubMessage getPubsubMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
+
+ Gson gson = new GsonBuilder().create();
+ Type listType = new TypeToken<List<RecordStatus>>() {}.getType();
+ JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
+ ByteString statusChangedTagsData = ByteString.copyFromUtf8(statusChangedTagsJson.toString());
+
+ PubsubMessage.Builder builder = PubsubMessage.newBuilder();
+ String tenant = headers.getPartitionId();
+ //This code it to provide backward compatibility to slb-account-id
+ if(!Strings.isNullOrEmpty(tenant)) {
+ builder.putAttributes(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionId());
+ } else {
+ builder.putAttributes(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ }
+
+ builder.putAttributes(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ builder.putAttributes( AppEngineHeaders.CLOUD_TRACE_CONTEXT, headers.getHeaders().get(AppEngineHeaders.CLOUD_TRACE_CONTEXT));
+ builder.setData(statusChangedTagsData);
+
+ return builder.build();
+ }
+
+ private Publisher getPublisher(String tenantName) throws IOException {
+ TenantInfo info = this.tenantStorageFactory.getTenantInfo(tenantName);
+ if (info == null) {
+ return null;
+ } else {
+ if (PUBSUB_CLIENTS.containsKey(tenantName)) return PUBSUB_CLIENTS.get(tenantName);
+
+ ProjectTopicName topicName = ProjectTopicName.newBuilder().setProject(info.getProjectId()).setTopic(TOPIC_ID).build();
+ Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(RETRY_SETTINGS).build();
+
+ if (publisher == null) return null;
+
+ PUBSUB_CLIENTS.put(tenantName, publisher);
+ return publisher;
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImpl.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImpl.java
new file mode 100644
index 000000000..4ed85ae29
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImpl.java
@@ -0,0 +1,115 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import com.google.common.base.Strings;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.model.SlbHeaders;
+import org.opendes.core.util.IHeadersInfo;
+import org.opendes.core.util.Preconditions;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpHeaders;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class HeadersInfoGcpImpl implements IHeadersInfo {
+
+ @Autowired
+ private HttpHeaders httpHeaders;
+
+ private DpsHeaders headersMap = null;
+
+ private static final HashSet<String> FORBIDDEN_FROM_LOGGING = new HashSet<>();
+ static {
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.AUTHORIZATION);
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.ON_BEHALF_OF);
+ }
+
+ private static final HashSet<String> FORWARDED_HEADERS = new HashSet<>();
+ static {
+ FORWARDED_HEADERS.add(AppEngineHeaders.CITY_LAT_LONG);
+ FORWARDED_HEADERS.add(AppEngineHeaders.COUNTRY);
+ FORWARDED_HEADERS.add(AppEngineHeaders.REGION);
+ FORWARDED_HEADERS.add(AppEngineHeaders.CITY);
+ FORWARDED_HEADERS.add(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT);
+ FORWARDED_HEADERS.add(AppEngineHeaders.TASK_QUEUE_NAME);
+ FORWARDED_HEADERS.add(AppEngineHeaders.DATA_GROUPS);
+ FORWARDED_HEADERS.add(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+ FORWARDED_HEADERS.add(AppEngineHeaders.TRACE_ID);
+ FORWARDED_HEADERS.add(AppEngineHeaders.CRON_SERVICE);
+ FORWARDED_HEADERS.add(SlbHeaders.PRIMARY_PARTITION_ID);
+ }
+
+ @Override
+ public DpsHeaders getHeaders() {
+ if (headersMap == null) {
+ headersMap = this.getCoreServiceHeaders(httpHeaders.toSingleValueMap());
+ }
+ return headersMap;
+ }
+
+ @Override
+ public String getUser() {
+ return getHeaders().getUserEmail();
+ }
+
+ @Override
+ public String getPartitionId() {
+ return getHeaders().getPartitionIdWithFallbackToAccountId();
+ }
+
+ @Override
+ public String getPrimaryPartitionId() {
+ return getHeadersMap().get(SlbHeaders.PRIMARY_PARTITION_ID);
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return getHeaders().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
+ Preconditions.checkNotNull(input, "input headers cannot be null");
+
+ DpsHeaders output = DpsHeaders.createFromMap(input);
+ input.forEach((key,value) -> {
+ if (FORWARDED_HEADERS.contains(key)) {
+ if (key.equals(AppEngineHeaders.CLOUD_TRACE_CONTEXT)) {
+ String traceContext = input.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+ if (!Strings.isNullOrEmpty(traceContext)) {
+ output.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
+ output.put(key, traceContext);
+ }
+ } else {
+ if (!output.getHeaders().containsKey(key.toLowerCase())) {
+ output.put(key, input.get(key));
+ }
+ }
+ }
+ });
+ output.addCorrelationIdIfMissing();
+ return output;
+ }
+
+ @Override
+ public String toString() {
+ return this.getHeadersMap().entrySet().stream().filter(map -> !FORBIDDEN_FROM_LOGGING.contains(map.getKey().toLowerCase())).map(Map.Entry::toString).collect(Collectors.joining(" | "));
+ }
+
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/RequestInfoImpl.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/RequestInfoImpl.java
new file mode 100644
index 000000000..edc5f15bb
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/RequestInfoImpl.java
@@ -0,0 +1,85 @@
+package org.opendes.indexer.gcp.util;
+
+import com.google.common.base.Strings;
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.Constants;
+import org.opendes.core.util.HeadersInfo;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+
+import static org.opendes.client.api.DpsHeaders.AUTHORIZATION;
+
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Autowired
+ private HeadersInfo headersInfo;
+ @Autowired
+ private ServiceAccountJwtGcpClientImpl serviceAccountJwtClient;
+
+ @Autowired
+ private TenantInfo tenantInfo;
+
+ private static final String expectedCronHeaderValue = "true";
+
+ @Override
+ public DpsHeaders getHeaders() {
+
+ return this.headersInfo.getHeaders();
+ }
+
+ @Override
+ public String getPartitionId() {
+ return this.headersInfo.getPartitionId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return this.headersInfo.getHeadersMap();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ DpsHeaders output = this.headersInfo.getHeaders();
+ output.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return output;
+ }
+
+ @Override
+ public boolean isCronRequest() {
+ String appEngineCronHeader = this.headersInfo.getHeadersMap().getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
+ return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
+ }
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ if (!this.headersInfo.getHeadersMap().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false;
+
+ String queueId = this.headersInfo.getHeadersMap().get(AppEngineHeaders.TASK_QUEUE_NAME);
+ return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
+ }
+
+ public String checkOrGetAuthorizationHeader() {
+ if (Config.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.headersInfo.getHeaders().getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
+ }
+ return authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+}
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImpl.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImpl.java
new file mode 100644
index 000000000..8e52738c4
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImpl.java
@@ -0,0 +1,181 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.exceptions.JWTDecodeException;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.ITenantFactory;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.cache.JwtCache;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.IdToken;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.IServiceAccountJwtClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
+
+ private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
+ private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
+
+ private final JsonFactory JSON_FACTORY = new JacksonFactory();
+
+ private Iam iam;
+
+ @Autowired
+ private ITenantFactory tenantInfoServiceProvider;
+ @Autowired
+ private HeadersInfoGcpImpl headersInfoGcp;
+ @Autowired
+ private JwtCache cacheService;
+ @Autowired
+ private JaxRsDpsLog log;
+
+ @Value("${GOOGLE_AUDIENCES}")
+ public String GOOGLE_AUDIENCES;
+
+ @Value("${INDEXER_HOST}")
+ public String INDEXER_HOST;
+
+ public String getIdToken(String tenantName) {
+ this.log.info("Tenant name received for auth token is: " + tenantName);
+ TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
+ if (tenant == null) {
+ this.log.error("Invalid tenant name receiving from pubsub");
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub");
+ }
+ try {
+
+ IdToken cachedToken = this.cacheService.get(tenant.getServiceAccount());
+ this.headersInfoGcp.getHeaders().put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
+
+ if (!IdToken.refreshToken(cachedToken)) {
+ return cachedToken.getTokenValue();
+ }
+
+ // Getting signed JWT
+ Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant);
+
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload));
+
+ String serviceAccountName = String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount());
+
+ Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest);
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ // Getting id token
+ List<NameValuePair> postParameters = new ArrayList<>();
+ postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
+ postParameters.add(new BasicNameValuePair("assertion", signedJwt));
+
+ HttpPost post = new HttpPost(JWT_AUDIENCE);
+ post.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType());
+ post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
+
+ try(CloseableHttpClient httpclient = HttpClients.createDefault();
+ CloseableHttpResponse httpResponse = httpclient.execute(post)) {
+ JsonObject jsonContent = new JsonParser().parse(EntityUtils.toString(httpResponse.getEntity())).getAsJsonObject();
+
+ if (!jsonContent.has("id_token")) {
+ log.error(String.format("Google IAM response: %s", jsonContent.toString()));
+ throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", "The user is not authorized to perform this action");
+ }
+
+ String token = jsonContent.get("id_token").getAsString();
+ IdToken idToken = IdToken.builder().tokenValue(token).expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
+
+ this.cacheService.put(tenant.getServiceAccount(), idToken);
+
+ return token;
+ }
+ } catch (JWTDecodeException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Invalid token, error decoding", e);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
+ }
+ }
+
+ public Iam getIam() throws Exception {
+
+ if (this.iam == null) {
+ HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
+
+ // Authenticate using Google Application Default Credentials.
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ List<String> scopes = new ArrayList<>();
+ // Enable full Cloud Platform scope.
+ scopes.add(IamScopes.CLOUD_PLATFORM);
+ credential = credential.createScoped(scopes);
+ }
+
+ // Create IAM API object associated with the authenticated transport.
+ this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential)
+ .setApplicationName(INDEXER_HOST)
+ .build();
+ }
+
+ return this.iam;
+ }
+
+ private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) {
+
+ Map<String, Object> payload = new HashMap<>();
+ String googleAudience = GOOGLE_AUDIENCES;
+ if (googleAudience.contains(",")) {
+ googleAudience = googleAudience.split(",")[0];
+ }
+ payload.put("target_audience", googleAudience);
+ payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
+ payload.put("iat", System.currentTimeMillis() / 1000);
+ payload.put("iss", tenantInfo.getServiceAccount());
+ payload.put("aud", JWT_AUDIENCE);
+
+ return payload;
+ }
+}
diff --git a/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/TraceIdExtractor.java b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/TraceIdExtractor.java
new file mode 100644
index 000000000..4d406e352
--- /dev/null
+++ b/indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/util/TraceIdExtractor.java
@@ -0,0 +1,63 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import com.google.common.base.Strings;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.springframework.util.MultiValueMap;
+
+import java.util.Random;
+import java.util.UUID;
+
+public class TraceIdExtractor {
+
+ /*
+ * "X-Cloud-Trace-Context: TRACE_ID/SPAN_ID;o=TRACE_TRUE" e.g. 105445aa7843bc8bf206b120001000/0;o=1"
+ * https://cloud.google.com/trace/docs/support
+ * */
+ public static String getTraceableCloudContext(MultiValueMap<String, String> requestHeaders) {
+ String traceContextHeader = requestHeaders.getFirst(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+
+ // get new if not found
+ if (Strings.isNullOrEmpty(traceContextHeader)) return getNewTraceContext();
+ // return as is
+ if (traceContextHeader.endsWith(";o=1")) return traceContextHeader;
+
+ String[] traceParts = traceContextHeader.split("[/;]");
+ // if there is only trace-id
+ if (traceParts.length == 1) return String.format("%s/%s;o=1", traceContextHeader, getNewSpanId());
+ // if trace-id and span-id
+ if (traceParts.length == 2) return String.format("%s;o=1", traceContextHeader);
+ // trace flag is turned off
+ return String.format("%s/%s;o=1", traceParts[0], traceParts[1]);
+ }
+
+ public static String getTraceId(String traceContextHeader) {
+ String[] traceParts = traceContextHeader.split("[/;]");
+ return traceParts.length > 0 ? traceParts[0] : getNewTraceId();
+ }
+
+ private static String getNewTraceContext() {
+ return String.format("%s/%s;o=1", getNewTraceId(), getNewSpanId());
+ }
+
+ private static String getNewTraceId() {
+ return UUID.randomUUID().toString().replaceAll("-", "");
+ }
+
+ private static String getNewSpanId() {
+ return Integer.toUnsignedString(new Random().nextInt());
+ }
+}
diff --git a/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImplTest.java b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImplTest.java
new file mode 100644
index 000000000..0c2bd705d
--- /dev/null
+++ b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/HeadersInfoGcpImplTest.java
@@ -0,0 +1,198 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.model.SlbHeaders;
+import org.opendes.core.util.HeadersUtil;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.springframework.http.HttpHeaders;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class HeadersInfoGcpImplTest {
+
+ @Mock
+ private HttpHeaders httpHeaders;
+ @InjectMocks
+ private HeadersInfoGcpImpl sut;
+
+ @Test
+ public void should_convert_Cloud_Trace_when_header_contains_it() {
+ Map<String, String> requestHeaders = new HashMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, "any token");
+ requestHeaders.put(AppEngineHeaders.CLOUD_TRACE_CONTEXT, "any trace");
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders);
+
+ assertEquals("any token", map.getAuthorization());
+ assertEquals("any trace", map.getHeaders().get(AppEngineHeaders.TRACE_ID));
+ }
+
+ @Test
+ public void should_return_header_logs_when_header_contains_email() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.add(DpsHeaders.AUTHORIZATION, "any token");
+ requestHeaders.add(DpsHeaders.ON_BEHALF_OF, "any onBehalf");
+ requestHeaders.add(DpsHeaders.CORRELATION_ID, "any correlationId");
+ requestHeaders.add(DpsHeaders.USER_EMAIL, "abc@xyz.com");
+ requestHeaders.add(DpsHeaders.ACCOUNT_ID, "any account");
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders.toSingleValueMap());
+ assertEquals("any token", map.getAuthorization());
+ DpsHeaders headers = DpsHeaders.createFromEntrySet(requestHeaders.entrySet());
+
+ assertEquals("account id: any account | on behalf: any onBehalf | user email: any onBehalf | correlation id: " +
+ "any correlationId", HeadersUtil.toLogMsg(headers, "any onBehalf"));
+ }
+
+ @Test
+ public void should_return_header_logs_when_jwt_decode_fails() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.add(DpsHeaders.AUTHORIZATION, "any token");
+ requestHeaders.add(DpsHeaders.ON_BEHALF_OF, "any onBehalf");
+ requestHeaders.add(DpsHeaders.CORRELATION_ID, "any correlationId");
+ requestHeaders.add(DpsHeaders.ACCOUNT_ID, "any account");
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders.toSingleValueMap());
+
+ assertEquals("any token", map.getAuthorization());
+
+ DpsHeaders headers = DpsHeaders.createFromEntrySet(requestHeaders.entrySet());
+ assertEquals("account id: any account | on behalf: any onBehalf | correlation id: any correlationId",
+ HeadersUtil.toLogMsg(headers, null));
+ }
+
+ @Test
+ public void should_return_header_logs_when_header_doesNot_contain_email() {
+ Map<String, String> requestHeaders = new HashMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, "any token");
+ requestHeaders.put(DpsHeaders.ON_BEHALF_OF, "any onBehalf");
+ requestHeaders.put(DpsHeaders.CORRELATION_ID, "any correlationId");
+ requestHeaders.put(DpsHeaders.ACCOUNT_ID, "any account");
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders);
+ assertEquals("any token", map.getHeaders().get(DpsHeaders.AUTHORIZATION));
+ }
+
+ @Test
+ public void check_correct_headers() {
+ Map<String, String> requestHeaders = new HashMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, "any token");
+ requestHeaders.put(DpsHeaders.CORRELATION_ID, "any correlationId");
+ requestHeaders.put(DpsHeaders.ACCOUNT_ID, "any account");
+ requestHeaders.put(DpsHeaders.ON_BEHALF_OF, "any onBehalf");
+ requestHeaders.put(DpsHeaders.USER_EMAIL, "abc@xyz.com");
+ requestHeaders.put(DpsHeaders.CONTENT_TYPE, "any contentType");
+ requestHeaders.put(AppEngineHeaders.DATA_GROUPS, "any dataGrp");
+ requestHeaders.put(AppEngineHeaders.CRON_SERVICE, "true");
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders);
+
+ assertEquals("any token", map.getAuthorization());
+ assertEquals("any correlationId", map.getCorrelationId());
+ assertEquals("abc@xyz.com", map.getUserEmail());
+ assertEquals("any account", map.getPartitionIdWithFallbackToAccountId());
+ assertEquals("any onBehalf", map.getOnBehalfOf());
+ assertEquals("any contentType", map.getHeaders().get(DpsHeaders.CONTENT_TYPE));
+ assertEquals("any dataGrp", map.getHeaders().get(AppEngineHeaders.DATA_GROUPS));
+ assertEquals("true", map.getHeaders().get(AppEngineHeaders.CRON_SERVICE));
+ }
+
+ @Test
+ public void should_return_null_auth_header_when_invalid_header() {
+ Map<String, String> requestHeaders = new HashMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, null);
+
+ DpsHeaders map = this.sut.getCoreServiceHeaders(requestHeaders);
+
+ assertNotNull(map);
+ assertNull(map.getAuthorization());
+ }
+
+ @Test
+ public void should_addCorrelationId_when_gettingHeaders() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.put(DpsHeaders.USER_EMAIL, singletonList("a@b.com"));
+ when(httpHeaders.toSingleValueMap()).thenReturn(requestHeaders.toSingleValueMap());
+
+ assertNotNull(sut.getHeaders().getCorrelationId());
+ }
+
+ @Test
+ public void should_returnUser_when_requested() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.put(DpsHeaders.USER_EMAIL, singletonList("a@b.com"));
+ when(httpHeaders.toSingleValueMap()).thenReturn(requestHeaders.toSingleValueMap());
+
+ assertEquals("a@b.com", sut.getUser());
+ }
+
+ @Test
+ public void should_returnPrimaryAccountId_when_requested() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.put(SlbHeaders.PRIMARY_PARTITION_ID, singletonList("apc"));
+ when(httpHeaders.toSingleValueMap()).thenReturn(requestHeaders.toSingleValueMap());
+
+ assertEquals("apc", sut.getPrimaryPartitionId());
+ }
+
+ @Test
+ public void should_convert_to_string_when_map_is_correct() {
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.add("a", "a val");
+ requestHeaders.add("b", "b val");
+ requestHeaders.add(DpsHeaders.AUTHORIZATION, "blah");
+ when(this.httpHeaders.toSingleValueMap()).thenReturn(requestHeaders.toSingleValueMap());
+
+ assertFalse(this.sut.toString().contains("a=a val"));
+ assertFalse(this.sut.toString().contains("b=b val"));
+ }
+
+// @Test
+// public void should_convert_multivalued_map_to_hash_map_when_input_map_is_correct() {
+// MultivaluedMap<String, String> requestHeaders = new MultivaluedMapImpl<String, String>();
+// requestHeaders.putSingle(DpsHeaders.AUTHORIZATION, "any token");
+// requestHeaders.putSingle(DpsHeaders.CORRELATION_ID, "any correlationId");
+// requestHeaders.putSingle(DpsHeaders.ACCOUNT_ID, "any account");
+// requestHeaders.putSingle(DpsHeaders.ON_BEHALF_OF, "any onBehalf");
+// requestHeaders.putSingle(DpsHeaders.USER_EMAIL, "abc@xyz.com");
+// requestHeaders.putSingle(DpsHeaders.CONTENT_TYPE, "any contentType");
+// requestHeaders.putSingle(AppEngineHeaders.DATA_GROUPS, "any dataGrp");
+//
+// Map<String, String> map = this.sut.convertMultiToRegularMap(requestHeaders);
+//
+// assertEquals("any token", map.get(DpsHeaders.AUTHORIZATION));
+// assertEquals("any correlationId", map.get(DpsHeaders.CORRELATION_ID));
+// assertEquals("abc@xyz.com", map.get(DpsHeaders.USER_EMAIL));
+// assertEquals("any account", map.get(DpsHeaders.ACCOUNT_ID));
+// assertEquals("any onBehalf", map.get(DpsHeaders.ON_BEHALF_OF));
+// assertEquals("any contentType", map.get(DpsHeaders.CONTENT_TYPE));
+// assertEquals("any dataGrp", map.get(AppEngineHeaders.DATA_GROUPS));
+// }
+}
\ No newline at end of file
diff --git a/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImplTest.java b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImplTest.java
new file mode 100644
index 000000000..ec298fe9b
--- /dev/null
+++ b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/ServiceAccountJwtGcpClientImplTest.java
@@ -0,0 +1,178 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.cache.JwtCache;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.model.IdToken;
+import org.opendes.core.service.TenantInfoServiceImpl;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, Config.class})
+public class ServiceAccountJwtGcpClientImplTest {
+
+ private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private GoogleCredential credential;
+ @Mock
+ private NetHttpTransport httpTransport;
+ @Mock
+ private SignJwtResponse signJwtResponse;
+ @Mock
+ private Iam iam;
+ @Mock
+ private Iam.Projects iamProject;
+ @Mock
+ private Iam.Projects.ServiceAccounts iamProjectServiceAccounts;
+ @Mock
+ private Iam.Projects.ServiceAccounts.SignJwt signJwt;
+ @Mock
+ private CloseableHttpClient httpClient;
+ @Mock
+ private CloseableHttpResponse httpResponse;
+ @InjectMocks
+ private TenantInfoServiceImpl tenantInfoServiceProvider;
+ @Mock
+ private TenantInfoServiceImpl tenantInfoService;
+ @Mock
+ private JwtCache cacheService;
+ @Mock
+ private HeadersInfoGcpImpl headersInfoGcp;
+ @InjectMocks @Spy
+ private ServiceAccountJwtGcpClientImpl sut;
+ @Before
+ public void setup() throws Exception {
+ initMocks(this);
+
+// mockStatic(GoogleNetHttpTransport.class);
+// mockStatic(GoogleCredential.class);
+// mockStatic(HttpClients.class);
+// mockStatic(EntityUtils.class);
+// mockStatic(Config.class);
+
+ when(GoogleNetHttpTransport.newTrustedTransport()).thenReturn(httpTransport);
+ when(GoogleCredential.getApplicationDefault()).thenReturn(credential);
+ when(credential.createScopedRequired()).thenReturn(true);
+ when(credential.createScoped(any())).thenReturn(credential);
+ when(HttpClients.createDefault()).thenReturn(httpClient);
+ when(httpClient.execute(any())).thenReturn(httpResponse);
+ when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+ when(Config.getGoogleAudiences()).thenReturn("aud");
+
+ when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService);
+
+ TenantInfo tenantInfo = new TenantInfo();
+ tenantInfo.setServiceAccount("tenant");
+ when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
+
+ when(this.sut.getIam()).thenReturn(iam);
+ when(this.iam.projects()).thenReturn(iamProject);
+ when(this.iamProject.serviceAccounts()).thenReturn(iamProjectServiceAccounts);
+ when(this.iamProjectServiceAccounts.signJwt(any(), any())).thenReturn(signJwt);
+ when(this.signJwt.execute()).thenReturn(signJwtResponse);
+ when(this.signJwtResponse.getSignedJwt()).thenReturn("testJwt");
+
+ Map<String, String> headers = new HashMap<>();
+ DpsHeaders dpsHeaders = DpsHeaders.createFromMap(headers);
+ when(this.headersInfoGcp.getHeaders()).thenReturn(dpsHeaders);
+ }
+
+ @Test
+ public void should_returnCachedToken_givenCachedToken_getIdTokenTest() {
+ String tokenValue = "tokenValue";
+ IdToken idToken = IdToken.builder().tokenValue(tokenValue).expirationTimeMillis(System.currentTimeMillis() + 10000000L).build();
+ when(this.cacheService.get(any())).thenReturn(idToken);
+
+ String returnedIdToken = this.sut.getIdToken(tokenValue);
+
+ Assert.assertEquals(tokenValue, returnedIdToken);
+ }
+
+ @Test
+ public void should_returnValidToken_getIdTokenTest() throws Exception {
+ when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", JWT_TOKEN));
+
+ String returnedToken = this.sut.getIdToken("tenant");
+
+ Assert.assertEquals(JWT_TOKEN, returnedToken);
+ }
+
+ @Test
+ public void should_return500_given_invalidJWTResponse_getIdTokenException() {
+ try {
+ when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", "invalid jwt"));
+
+ this.sut.getIdToken("tenant");
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("Invalid token, error decoding", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return403_given_missingIdTokenResponse_getIdTokenException() {
+ try {
+ when(EntityUtils.toString(any())).thenReturn("{}");
+
+ this.sut.getIdToken("tenant");
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode());
+ Assert.assertEquals("The user is not authorized to perform this action", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/TraceIdExtractorTest.java b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/TraceIdExtractorTest.java
new file mode 100644
index 000000000..09cf784b6
--- /dev/null
+++ b/indexer-service-gcp/src/test/java/org/opendes/indexer/gcp/util/TraceIdExtractorTest.java
@@ -0,0 +1,71 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.util;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.indexer.gcp.model.AppEngineHeaders;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static junit.framework.TestCase.assertTrue;
+
+@RunWith(SpringRunner.class)
+public class TraceIdExtractorTest {
+
+ @Test
+ public void should_getTraceableCloudContext_when_header_doesNot_contains_it() {
+ List<String> token = new ArrayList<>();
+ token.add("any token");
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, token);
+
+ String actual = TraceIdExtractor.getTraceableCloudContext(requestHeaders);
+
+ assertTrue(actual.matches(".*o=1"));
+ }
+
+ @Test
+ public void should_getTraceableCloudContext_when_header_contains_it() {
+ List<String> token = new ArrayList<>();
+ token.add("any token");
+ List<String> trace = new ArrayList<>();
+ trace.add("any trace");
+ MultiValueMap<String, String> requestHeaders = new LinkedMultiValueMap<>();
+ requestHeaders.put(DpsHeaders.AUTHORIZATION, token);
+ requestHeaders.put(AppEngineHeaders.CLOUD_TRACE_CONTEXT, trace);
+
+ String actual = TraceIdExtractor.getTraceableCloudContext(requestHeaders);
+ assertTrue(actual.matches("any trace/.*o=1"));
+
+ trace = new ArrayList<>();
+ trace.add("any/trace");
+ requestHeaders.put(AppEngineHeaders.CLOUD_TRACE_CONTEXT, trace);
+ actual = TraceIdExtractor.getTraceableCloudContext(requestHeaders);
+ assertTrue(actual.matches("any/trace;o=1"));
+
+ trace = new ArrayList<>();
+ trace.add("any/trace/test");
+ requestHeaders.put(AppEngineHeaders.CLOUD_TRACE_CONTEXT, trace);
+ actual = TraceIdExtractor.getTraceableCloudContext(requestHeaders);
+ assertTrue(actual.matches("any/trace;o=1"));
+ }
+
+}
diff --git a/indexer-service-gcp/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/indexer-service-gcp/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 000000000..ca6ee9cea
--- /dev/null
+++ b/indexer-service-gcp/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file
diff --git a/indexer-service-root/maven/settings.xml b/indexer-service-root/maven/settings.xml
new file mode 100644
index 000000000..504ef0e8d
--- /dev/null
+++ b/indexer-service-root/maven/settings.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <servers>
+ <server>
+ <id>dev-azure-com-slb-des-ext-collaboration-os-core</id>
+ <username>os-core</username>
+ <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
+ <!-- The generated token expires on or before 11/14/2019 -->
+ <password>${VSTS_FEED_TOKEN}</password>
+ </server>
+ </servers>
+</settings>
diff --git a/indexer-service-root/mvnw b/indexer-service-root/mvnw
new file mode 100644
index 000000000..8b9da3b8b
--- /dev/null
+++ b/indexer-service-root/mvnw
@@ -0,0 +1,286 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven2 Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# M2_HOME - location of maven2's installed home dir
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+ if [ -f /etc/mavenrc ] ; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ] ; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ MINGW*) mingw=true;;
+ Darwin*) darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ export JAVA_HOME="`/usr/libexec/java_home`"
+ else
+ export JAVA_HOME="/Library/Java/Home"
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=`java-config --jre-home`
+ fi
+fi
+
+if [ -z "$M2_HOME" ] ; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+
+ saveddir=`pwd`
+
+ M2_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ M2_HOME=`cd "$M2_HOME" && pwd`
+
+ cd "$saveddir"
+ # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --unix "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME="`(cd "$M2_HOME"; pwd)`"
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
+ # TODO classpath?
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="`which javac`"
+ if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=`which readlink`
+ if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
+ if $darwin ; then
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
+ else
+ javaExecutable="`readlink -f \"$javaExecutable\"`"
+ fi
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaHome=`expr "$javaHome" : '\(.*\)/bin'`
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="`which java`"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+ if [ -z "$1" ]
+ then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ] ; do
+ if [ -d "$wdir"/.mvn ] ; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=`cd "$wdir/.."; pwd`
+ fi
+ # end of workaround
+ done
+ echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ echo "$(tr -s '\n' ' ' < "$1")"
+ fi
+}
+
+BASE_DIR=`find_maven_basedir "$(pwd)"`
+if [ -z "$BASE_DIR" ]; then
+ exit 1;
+fi
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found .mvn/wrapper/maven-wrapper.jar"
+ fi
+else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
+ fi
+ jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"
+ while IFS="=" read key value; do
+ case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
+ esac
+ done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Downloading from: $jarUrl"
+ fi
+ wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
+
+ if command -v wget > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found wget ... using wget"
+ fi
+ wget "$jarUrl" -O "$wrapperJarPath"
+ elif command -v curl > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found curl ... using curl"
+ fi
+ curl -o "$wrapperJarPath" "$jarUrl"
+ else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Falling back to using Java to download"
+ fi
+ javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ if [ -e "$javaClass" ]; then
+ if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Compiling MavenWrapperDownloader.java ..."
+ fi
+ # Compiling the Java class
+ ("$JAVA_HOME/bin/javac" "$javaClass")
+ fi
+ if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ # Running the downloader
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Running MavenWrapperDownloader.java ..."
+ fi
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+if [ "$MVNW_VERBOSE" = true ]; then
+ echo $MAVEN_PROJECTBASEDIR
+fi
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --path --windows "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
+fi
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/indexer-service-root/mvnw.cmd b/indexer-service-root/mvnw.cmd
new file mode 100644
index 000000000..fef5a8f7f
--- /dev/null
+++ b/indexer-service-root/mvnw.cmd
@@ -0,0 +1,161 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM https://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven2 Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
+if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"
+FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO (
+ IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+ echo Found %WRAPPER_JAR%
+) else (
+ echo Couldn't find %WRAPPER_JAR%, downloading it ...
+ echo Downloading from: %DOWNLOAD_URL%
+ powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"
+ echo Finished downloading %WRAPPER_JAR%
+)
+@REM End of extension
+
+%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
+if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%" == "on" pause
+
+if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
+
+exit /B %ERROR_CODE%
diff --git a/indexer-service-root/pom.xml b/indexer-service-root/pom.xml
new file mode 100644
index 000000000..6b0143bdb
--- /dev/null
+++ b/indexer-service-root/pom.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>indexer-service-root</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>indexer-service-root</name>
+ <description>Indexer Service Root</description>
+ <packaging>jar</packaging>
+
+ <!--<repositories>
+ <repository>
+ <id>elastic</id>
+ <url>https://artifacts.elastic.co/maven</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>-->
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opendes.core</groupId>
+ <artifactId>indexer-search-core-root</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <!-- spring boot dependencies -->
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>9.0.21</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-tomcat</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-jersey</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-actuator</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-validation</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.1-jre</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.3</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.12</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.9.9.3</version>
+ </dependency>
+ <dependency>
+ <groupId>io.jsonwebtoken</groupId>
+ <artifactId>jjwt</artifactId>
+ <version>0.9.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/commons-lang/commons-lang -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+
+ <!-- swagger dependencies -->
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-swagger2</artifactId>
+ <version>${springfox-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-swagger-ui</artifactId>
+ <version>${springfox-version}</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/SwaggerDoc.java b/indexer-service-root/src/main/java/org/opendes/indexer/SwaggerDoc.java
new file mode 100644
index 000000000..d0c9a387a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/SwaggerDoc.java
@@ -0,0 +1,148 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer;
+
+public final class SwaggerDoc {
+
+ // GENERAL
+ public static final String RESPONSE_BAD_GATEWAY = "Search service scale-up is taking longer than expected. Wait 10 seconds and retry.";
+
+ // SEARCH
+ public static final String SEARCH_TAG = "Search";
+ public static final String SEARCH_DESCRIPTION = "Service endpoints to search data in Data Ecosystem";
+
+ // QUERY
+ public static final String QUERY_POST_TITLE = "Queries using the input request criteria.";
+ public static final String QUERY_OPERATION_ID = "Query";
+ public static final String QUERY_POST_NOTES = "The API supports full text search on string fields, range queries on date, numeric or string fields, along with geo-spatial search. Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins' or 'users.datalake.ops'. In addition, users must be a member of data groups to access the data.";
+ public static final String QUERY_POST_RESPONSE_OK = "Success";
+ public static final String QUERY_POST_RESPONSE_BAD_REQUEST = "Invalid parameters were given on request";
+ public static final String QUERY_POST_RESPONSE_NOT_AUTHORIZED = "User not authorized to perform the action";
+
+ // CCS QUERY
+ public static final String CCS_QUERY_OPERATION_ID = "CCS Query";
+ public static final String CCS_QUERY_NOTES = "The API supports cross cluster searches when given the list of partitions.";
+
+ // QUERY_WITH_CURSOR
+ public static final String QUERY_WITH_CURSOR_POST_TITLE = "Queries using the input request criteria.";
+ public static final String QUERY_WITH_CURSOR_OPERATION_ID = "Query with cursor";
+ public static final String QUERY_WITH_CURSOR_POST_NOTES = "The API supports full text search on string fields, range queries on date, numeric or string fields, along with geo-spatial search. Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins' or 'users.datalake.ops'. In addition, users must be a member of data groups to access the data. It can be used to retrieve large numbers of results (or even all results) from a single search request, in much the same way as you would use a cursor on a traditional database.";
+ public static final String QUERY_WITH_CURSOR_POST_RESPONSE_OK = "Success";
+ public static final String QUERY_WITH_CURSOR_POST_RESPONSE_BAD_REQUEST = "Invalid parameters were given on request";
+ public static final String QUERY_WITH_CURSOR_POST_RESPONSE_NOT_AUTHORIZED = "User not authorized to perform the action";
+
+ // SCHEMA
+ public static final String INDEX_SCHEMA_GET_TITLE = "Returns the index schema for given 'kind'.";
+ public static final String INDEX_SCHEMA_OPERATION_ID = "Get index schema";
+ public static final String INDEX_SCHEMA_GET_NOTES = "The API returns the schema for a given kind which is used find what attributes are indexed and their respective data types (at index time). Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins' or 'users.datalake.ops'";
+ public static final String INDEX_SCHEMA_GET_RESPONSE_OK = "Success";
+ public static final String INDEX_SCHEMA_GET_RESPONSE_BAD_REQUEST = "Invalid parameters were given on request";
+ public static final String INDEX_SCHEMA_GET_RESPONSE_NOT_FOUND = "Index schema for requested kind not found";
+
+ // DELETE INDEX
+ public static final String INDEX_DELETE_TITLE = "Deletes all documents from index for given 'kind'.";
+ public static final String INDEX_DELETE_OPERATION_ID = "Delete index";
+ public static final String INDEX_DELETE_NOTES = "The API can be used to purge all indexed documents for a kind. Required roles: 'users.datalake.admins' or 'users.datalake.ops'";
+ public static final String INDEX_DELETE_RESPONSE_NO_CONTENT = "No Content Returned";
+ public static final String INDEX_DELETE_RESPONSE_BAD_REQUEST = "Invalid parameters were given on request";
+ public static final String INDEX_DELETE_RESPONSE_NOT_FOUND = "Requested kind not found";
+ public static final String INDEX_DELETE_RESPONSE_CONFLICT = "Unable to delete the index because it is currently locked";
+
+ // SEARCH SERVICE PARAMETERS
+ public static final String PARAMETER_KIND = "Kind of the record.";
+ public static final String PARAMETER_ACCOUNT_ID = "Account ID is the active DELFI account (SLB account or customer's account) which the users choose to use with the Search API.";
+ public static final String PARAMETER_ONBEHALF_ACCOUNT_ID = "Token (Google ID or SAuth) representing the user whose request is sent on behalf of.";
+
+ // REQUEST, RESPONSE & VALIDATIONS
+ // QUERIES
+ public static final String KIND_REQUEST_DESCRIPTION = "'kind' to search";
+ public static final String KIND_VALIDATION_CAN_NOT_BE_NULL_OR_EMPTY = "'kind' can not be null or empty";
+ public static final String KIND_EXAMPLE = "common:ihs:well:1.0.0";
+ public static final String LIMIT_VALIDATION_MIN_MSG = "'limit' must be equal or greater than 0";
+ public static final String OFFSET_VALIDATION_MIN_MSG = "'offset' must be equal or greater than 0";
+ public static final String SORT_FIELD_VALIDATION_NOT_EMPTY_MSG = "'sort.field' can not be null or empty";
+ public static final String SORT_FIELD_LIST_VALIDATION_NOT_EMPTY_MSG = "'sort.field' list can not have null or empty values";
+ public static final String SORT_NOT_VALID_ORDER_OPTION = "Not a valid order option. It can only be either 'ASC' or 'DESC'";
+ public static final String SORT_ORDER_VALIDATION_NOT_EMPTY_MSG = "'sort.order' can not be null or empty";
+ public static final String SORT_FIELD_ORDER_SIZE_NOT_MATCH = "'sort.field' and 'sort.order' size do not match";
+ public static final String LIMIT_DESCRIPTION = "The maximum number of results to return from the given offset. If no limit is provided, then it will return 10 items. Max number of items which can be fetched by the query is 100. (If you wish to fetch large set of items, please use query_with_cursor API)";
+ public static final String QUERY_DESCRIPTION = "The query string in Lucene query string syntax.";
+ public static final String SORT_DESCRIPTION = "The fields and orders to return sorted results.";
+ public static final String SORT_FIELD_DESCRIPTION = "The list of fields to sort the results.";
+ public static final String SORT_ORDER_DESCRIPTION = "The list of orders to sort the results. The element must be either ASC or DESC.";
+ public static final String RETURNED_FIELDS_DESCRIPTION = "The fields on which to project the results.";
+ public static final String OFFSET_DESCRIPTION = "The starting offset from which to return results.";
+ public static final String CURSOR_DESCRIPTION = "Search context to retrieve next batch of results.";
+ public static final String AGGREGATEBY_DESCRIPTION = "The aggregateBy field returns the distinct values of the given field.";
+ public static final String QUERYASOWNER_DESCRIPTION = "The queryAsOwner switches between viewer and owner to return results that you are entitled to view or results you are the owner of.";
+
+ // SPATIAL FILTER
+ public static final String FIELD_VALIDATION_NON_NULL_MSG = "'spatialFilter.field' can not be null";
+ public static final String TOPLEFT_VALIDATION_NON_NULL_MSG = "'byBoundingBox.topLeft' can not be null";
+ public static final String BOTTOMRIGHT_VALIDATION_NON_NULL_MSG = "'byBoundingBox.bottomRight' can not be null";
+ public static final String DISTANCE_VALIDATION_MIN_MSG = "'distance' must be greater than 0";
+ public static final String DISTANCE_VALIDATION_MAX_MSG = "'distance' cannot be greater than 1.5E203";
+ public static final String DISTANCE_POINT_VALIDATION_NON_NULL_MSG = "'byDistance.point' can not be null";
+ public static final String GEOPOLYGON_POINT_VALIDATION_NON_NULL_MSG = "'byGeoPolygon.point' list can not be null or empty";
+ public static final String LATITUDE_VALIDATION_RANGE_MSG = "'latitude' value is out of the range [-90, 90]";
+ public static final String LONGITUDE_VALIDATION_RANGE_MSG = "'longitude' value is out of the range [-180, 180]";
+ public static final String FIELD_DESCRIPTION = "geo-point field in the index on which filtering will be performed. Use GET schema API to find which fields supports spatial search.";
+ public static final String SPATIAL_FILTER_DESCRIPTION = "A spatial filter to apply.";
+ public static final String QUERY_BY_BOUNDING_BOX_DESCRIPTION = "A query allowing to filter hits based on a point location within a bounding box.";
+ public static final String QUERY_BY_DISTANCE_DESCRIPTION = "Filters documents that include only hits that exist within a specific distance from a geo point.";
+ public static final String QUERY_BY_GEO_POLYGON_DESCRIPTION = "A query allowing to filter hits that only fall within a polygon of points.";
+ public static final String TOPLEFT_BOUNDING_BOX_DESCRIPTION = "Top left corner of the bounding box.";
+ public static final String BOTTOMRIGHT_BOUNDING_BOX_DESCRIPTION = "Bottom right corner of the bounding box.";
+ public static final String DISTANCE_DESCRIPTION = "The radius of the circle centered on the specified location. Points which fall into this circle are considered to be matches.";
+ public static final String POINT_DISTANCE_DESCRIPTION = "Center point of the query.";
+ public static final String POINTS_GEO_POLYGON_DESCRIPTION = "Polygon defined by a set of points.";
+ public static final String LATITUDE = "Latitude of point.";
+ public static final String LONGITUDE = "Longitude of point.";
+
+ // INDEXER
+ public static final String INDEXER_TAG = "Indexer";
+ public static final String INDEXER_DESCRIPTION = "Indexer endpoints to index data in Data Ecosystem";
+
+ // REINDEX
+ public static final String REINDEX_POST_TITLE = "Re-index given 'kind'.";
+ public static final String REINDEX_OPERATION_ID = "Reindex kind";
+ public static final String REINDEX_POST_NOTES = "The API triggers re-indexing of 'kind'. Required roles: 'users.datalake.admins' or 'users.datalake.ops'";
+ public static final String REINDEX_POST_RESPONSE_OK = "Success";
+ public static final String REINDEX_POST_RESPONSE_BAD_REQUEST = "Invalid parameters were given on request";
+ public static final String REINDEX_POST_RESPONSE_NOT_FOUND = "Requested 'kind' not found";
+
+ // TASK STATUS
+ public static final String COPY_TASK_GET_TITLE = "Get status of task running for the tenant.";
+ public static final String COPY_TASK_OPERATION_ID = "Task status";
+ public static final String COPY_TASK_GET_NOTES = "Get status of task running for the tenant. Required roles: 'users.datalake.admins' or 'users.datalake.ops'";
+ public static final String COPY_TASK_GET_RESPONSE_OK = "Success";
+
+ // COPY INDEX
+ public static final String COPY_INDEX_GET_TITLE = "Copies index for kind from 'common' tenant to private tenant.";
+ public static final String COPY_INDEX_OPERATION_ID = "Copy index";
+ public static final String COPY_INDEX_POST_NOTES = "Copies index for kind from 'common' tenant to private tenant. Required roles: 'users.datalake.admins' or 'users.datalake.ops'";
+ public static final String COPY_INDEX_POST_RESPONSE_OK = "Success";
+
+ // INDEXER SERVICE PARAMETERS
+ public static final String PARAMETER_TASK_ID = "Task id.";
+
+ // SHARED AUTH
+ public static final String BEARER_AUTH = "Bearer";
+ public static final String GOOGLE_ID_AUTH = "google_id_token";
+ public static final String SAUTH_ID_AUTH = "sauth_id_token";
+
+ // REQUEST VALIDATION
+ public static final String REQUEST_VALIDATION_NOT_NULL_BODY = "Request body can not be null";
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/api/HealthCheckApi.java b/indexer-service-root/src/main/java/org/opendes/indexer/api/HealthCheckApi.java
new file mode 100644
index 000000000..47fcc963a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/api/HealthCheckApi.java
@@ -0,0 +1,40 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.api;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.security.PermitAll;
+
+@RestController
+@RequestMapping("/_ah")
+public class HealthCheckApi {
+
+ @PermitAll
+ @GetMapping("/liveness_check")
+ public ResponseEntity<String> livenessCheck(){
+ return new ResponseEntity<String>("Indexer service is alive", org.springframework.http.HttpStatus.OK);
+ }
+
+ @PermitAll
+ @GetMapping("/readiness_check")
+ public ResponseEntity<String> readinessCheck() {
+ return new ResponseEntity<String>("Legal service is ready", HttpStatus.OK);
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/api/RecordIndexerApi.java b/indexer-service-root/src/main/java/org/opendes/indexer/api/RecordIndexerApi.java
new file mode 100644
index 000000000..ee4bc6144
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/api/RecordIndexerApi.java
@@ -0,0 +1,70 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.api;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+
+import io.swagger.annotations.ApiOperation;
+import org.opendes.core.model.RecordChangedMessages;
+import org.opendes.indexer.SwaggerDoc;
+import org.opendes.indexer.util.JobStatus;
+import org.springframework.http.HttpStatus;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.service.IndexerService;
+import org.opendes.indexer.service.ReindexService;
+import org.opendes.indexer.util.RecordInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+import java.lang.reflect.Type;
+import java.util.List;
+
+@RestController
+@RequestMapping("/_dps/task-handlers")
+public class RecordIndexerApi {
+
+ @Autowired
+ private IndexerService indexerService;
+ @Autowired
+ private ReindexService reIndexService;
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/index-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<JobStatus> indexWorker (
+ @Valid @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @RequestParam RecordChangedMessages recordChangedMessages) throws Exception {
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
+ return new ResponseEntity(indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos),HttpStatus.OK);
+ }
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/reindex-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity reindex(
+ @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid RecordReindexRequest recordReindexRequest) {
+
+ return new ResponseEntity(reIndexService.reindexRecords(recordReindexRequest),HttpStatus.OK);
+ }
+}
\ No newline at end of file
diff --git a/dps-indexer/src/main/java/com/slb/indexer/api/ReindexApi.java b/indexer-service-root/src/main/java/org/opendes/indexer/api/ReindexApi.java
similarity index 60%
rename from dps-indexer/src/main/java/com/slb/indexer/api/ReindexApi.java
rename to indexer-service-root/src/main/java/org/opendes/indexer/api/ReindexApi.java
index 9a3bc00cf..1643f29f8 100644
--- a/dps-indexer/src/main/java/com/slb/indexer/api/ReindexApi.java
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/api/ReindexApi.java
@@ -12,51 +12,46 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.slb.indexer.api;
+package org.opendes.indexer.api;
-import com.slb.core.SwaggerDoc;
-import com.slb.core.model.SearchServiceRole;
-import com.slb.core.util.AppError;
-import com.slb.de.core.api.DpsHeaders;
-import com.slb.dps.multitenancy.TenantInfo;
-import com.slb.indexer.logging.AuditLogger;
-import com.slb.indexer.model.RecordReindexRequest;
-import com.slb.indexer.service.ReindexService;
import io.swagger.annotations.*;
-import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.SearchServiceRole;
+import org.opendes.core.util.AppError;
+import org.opendes.indexer.SwaggerDoc;
+import org.opendes.indexer.logging.AuditLogger;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.service.ReindexService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
import javax.annotation.security.RolesAllowed;
-import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
import static java.util.Collections.singletonList;
-
@Api(
value = SwaggerDoc.INDEXER_TAG,
authorizations = {@Authorization(value = SwaggerDoc.BEARER_AUTH),
@Authorization(value = SwaggerDoc.GOOGLE_ID_AUTH),
@Authorization(value = SwaggerDoc.SAUTH_ID_AUTH)})
-@Path("/reindex")
-@Consumes(MediaType.APPLICATION_JSON)
-@Produces(MediaType.APPLICATION_JSON)
+@RestController
+@RequestMapping("/reindex")
@ApiImplicitParams({
@ApiImplicitParam(name = DpsHeaders.ACCOUNT_ID, value = SwaggerDoc.PARAMETER_ACCOUNT_ID, required = true, defaultValue = TenantInfo.COMMON, dataType = "string", paramType = "header"),
@ApiImplicitParam(name = DpsHeaders.ON_BEHALF_OF, value = SwaggerDoc.PARAMETER_ONBEHALF_ACCOUNT_ID, dataType = "string", paramType = "header")})
public class ReindexApi {
- @Inject
+ @Autowired
private ReindexService reIndexService;
- @Inject
+ @Autowired
private AuditLogger auditLogger;
- @POST
+
@RolesAllowed({SearchServiceRole.ADMIN})
@ApiOperation(
value = SwaggerDoc.REINDEX_POST_TITLE,
@@ -64,20 +59,22 @@ public class ReindexApi {
notes = SwaggerDoc.REINDEX_POST_NOTES)
@ApiResponses({
@ApiResponse(
- code = HttpStatus.SC_OK,
+ code = org.apache.http.HttpStatus.SC_OK,
message = SwaggerDoc.REINDEX_POST_RESPONSE_OK,
response = String.class),
@ApiResponse(
- code = HttpStatus.SC_BAD_REQUEST,
+ code = org.apache.http.HttpStatus.SC_BAD_REQUEST,
message = SwaggerDoc.REINDEX_POST_RESPONSE_BAD_REQUEST,
response = AppError.class),
@ApiResponse(
- code = HttpStatus.SC_NOT_FOUND,
+ code = org.apache.http.HttpStatus.SC_NOT_FOUND,
message = SwaggerDoc.REINDEX_POST_RESPONSE_NOT_FOUND,
response = AppError.class)})
- public Response reindex(@NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY) @Valid RecordReindexRequest recordReindexRequest) {
- this.reIndexService.reindexRecords(recordReindexRequest);
- this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
- return Response.status(HttpStatus.SC_OK).type(MediaType.APPLICATION_JSON).build();
+ public ResponseEntity reindex(
+ @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid RecordReindexRequest recordReindexRequest) {
+ reIndexService.reindexRecords(recordReindexRequest);
+ auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
+ return new ResponseEntity (reIndexService.reindexRecords(recordReindexRequest), org.springframework.http.HttpStatus.OK);
}
}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/cache/SchemaCache.java b/indexer-service-root/src/main/java/org/opendes/indexer/cache/SchemaCache.java
new file mode 100644
index 000000000..29dc13dbe
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/cache/SchemaCache.java
@@ -0,0 +1,25 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.cache;
+
+import org.opendes.client.cache.RedisCache;
+import org.opendes.core.util.Config;
+
+public class SchemaCache extends RedisCache<String, String> {
+
+ public SchemaCache() {
+ super(Config.getSearchRedisHost(), Config.getSearchRedisPort(), Config.getSchemaCacheExpiration() * 60, String.class, String.class);
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditEvents.java b/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditEvents.java
new file mode 100644
index 000000000..6dccecc89
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditEvents.java
@@ -0,0 +1,221 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import com.google.common.base.Strings;
+import org.opendes.client.logging.payload.AuditAction;
+import org.opendes.client.logging.payload.AuditPayload;
+import org.opendes.client.logging.payload.AuditStatus;
+
+
+import java.util.List;
+
+import static org.opendes.client.logging.payload.AuditPayload.builder;
+
+public class AuditEvents {
+
+ private static final String INDEX_CREATE_RECORD_ACTION_ID = "IN001";
+ private static final String INDEX_CREATE_RECORDS_SUCCESS = "Successfully created record in index";
+ private static final String INDEX_CREATE_RECORDS_FAILURE = "Failed creating record in index";
+
+ private static final String INDEX_UPDATE_RECORD_ACTION_ID = "IN002";
+ private static final String INDEX_UPDATE_RECORDS_SUCCESS = "Successfully updated record in index";
+ private static final String INDEX_UPDATE_RECORDS_FAILURE = "Failed updating record in index";
+
+ private static final String INDEX_DELETE_RECORD_ACTION_ID = "IN003";
+ private static final String INDEX_DELETE_RECORDS_SUCCESS = "Successfully deleted record in index";
+ private static final String INDEX_DELETE_RECORDS_FAILURE = "Failed deleting record in index";
+
+ private static final String INDEX_PURGE_RECORD_ACTION_ID = "IN004";
+
+ private static final String REINDEX_KIND_ACTION_ID = "IN007";
+ private static final String REINDEX_KIND_OPERATION = "Reindex kind";
+
+ private static final String COPY_INDEX_ACTION_ID = "IN008";
+ private static final String COPY_INDEX_OPERATION = "Copy index";
+
+ private static final String GET_TASK_STATUS_ACTION_ID = "IN009";
+ private static final String GET_TASK_STATUS_OPERATION = "Get task status";
+
+ private static final String RUN_JOB_ACTION_ID = "IN010";
+ private static final String RUN_JOB_MESSAGE_SUCCESS = "Index clean-up status job run success";
+
+ private static final String INDEX_MAPPING_UPDATE_ACTION_ID = "IN0011";
+ private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully updated index mapping";
+ private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed updating index mapping";
+
+ private final String user;
+
+ public AuditEvents(String user) {
+ if (Strings.isNullOrEmpty(user)) {
+ throw new IllegalArgumentException("User not provided for audit events.");
+ }
+ this.user = user;
+ }
+
+ public AuditPayload getIndexCreateRecordSuccessEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.CREATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(INDEX_CREATE_RECORD_ACTION_ID)
+ .message(INDEX_CREATE_RECORDS_SUCCESS)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexCreateRecordFailEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.CREATE)
+ .status(AuditStatus.FAILURE)
+ .actionId(INDEX_CREATE_RECORD_ACTION_ID)
+ .message(INDEX_CREATE_RECORDS_FAILURE)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexUpdateRecordSuccessEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.UPDATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(INDEX_UPDATE_RECORD_ACTION_ID)
+ .message(INDEX_UPDATE_RECORDS_SUCCESS)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexUpdateRecordFailEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.UPDATE)
+ .status(AuditStatus.FAILURE)
+ .actionId(INDEX_UPDATE_RECORD_ACTION_ID)
+ .message(INDEX_UPDATE_RECORDS_FAILURE)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexDeleteRecordSuccessEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.DELETE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(INDEX_DELETE_RECORD_ACTION_ID)
+ .message(INDEX_DELETE_RECORDS_SUCCESS)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexDeleteRecordFailEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.DELETE)
+ .status(AuditStatus.FAILURE)
+ .actionId(INDEX_DELETE_RECORD_ACTION_ID)
+ .message(INDEX_DELETE_RECORDS_FAILURE)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexPurgeRecordSuccessEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.DELETE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(INDEX_PURGE_RECORD_ACTION_ID)
+ .message(INDEX_DELETE_RECORDS_SUCCESS)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexPurgeRecordFailEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.DELETE)
+ .status(AuditStatus.FAILURE)
+ .actionId(INDEX_PURGE_RECORD_ACTION_ID)
+ .message(INDEX_DELETE_RECORDS_FAILURE)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getReindexEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.CREATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(REINDEX_KIND_ACTION_ID)
+ .message(REINDEX_KIND_OPERATION)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getCopyIndexEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.CREATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(COPY_INDEX_ACTION_ID)
+ .message(COPY_INDEX_OPERATION)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getTaskStatusEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.READ)
+ .status(AuditStatus.SUCCESS)
+ .actionId(GET_TASK_STATUS_ACTION_ID)
+ .message(GET_TASK_STATUS_OPERATION)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
+ public AuditPayload getIndexCleanUpJobRunEvent(List<String> resources) {
+ return builder()
+ .action(AuditAction.JOB_RUN)
+ .status(AuditStatus.SUCCESS)
+ .user(this.user)
+ .actionId(RUN_JOB_ACTION_ID)
+ .message(RUN_JOB_MESSAGE_SUCCESS)
+ .resources(resources)
+ .build();
+ }
+
+ public AuditPayload getIndexMappingUpdateEvent(List<String> resources, boolean isSuccess) {
+ if (isSuccess) {
+ return builder()
+ .action(AuditAction.UPDATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(INDEX_MAPPING_UPDATE_ACTION_ID)
+ .message(INDEX_MAPPING_UPDATE_SUCCESS)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ } else {
+ return builder()
+ .action(AuditAction.UPDATE)
+ .status(AuditStatus.FAILURE)
+ .actionId(INDEX_MAPPING_UPDATE_ACTION_ID)
+ .message(INDEX_MAPPING_UPDATE_FAILURE)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditLogger.java b/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditLogger.java
new file mode 100644
index 000000000..eb4f16632
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/logging/AuditLogger.java
@@ -0,0 +1,100 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import org.opendes.client.logging.payload.AuditPayload;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.util.HeadersInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+@Component
+public class AuditLogger {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+ @Autowired
+ private HeadersInfo headers;
+
+ private AuditEvents events = null;
+
+ private AuditEvents getAuditEvents() {
+ if (this.events == null) {
+ this.events = new AuditEvents(this.headers.getUser());
+ }
+ return this.events;
+ }
+
+ public void indexCreateRecordSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexCreateRecordSuccessEvent(resources));
+ }
+
+ public void indexCreateRecordFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexCreateRecordFailEvent(resources));
+ }
+
+ public void indexUpdateRecordSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexUpdateRecordSuccessEvent(resources));
+ }
+
+ public void indexUpdateRecordFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexUpdateRecordFailEvent(resources));
+ }
+
+ public void indexDeleteRecordSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexDeleteRecordSuccessEvent(resources));
+ }
+
+ public void indexDeleteRecordFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexDeleteRecordFailEvent(resources));
+ }
+
+ public void indexPurgeRecordSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexPurgeRecordSuccessEvent(resources));
+ }
+
+ public void indexPurgeRecordFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexPurgeRecordFailEvent(resources));
+ }
+
+ public void getReindex(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getReindexEvent(resources));
+ }
+
+ public void copyIndex(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getCopyIndexEvent(resources));
+ }
+
+ public void getTaskStatus(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getTaskStatusEvent(resources));
+ }
+
+ public void getIndexCleanUpJobRun(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexCleanUpJobRunEvent(resources));
+ }
+
+ public void indexMappingUpdateSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,true));
+ }
+ public void indexMappingUpdateFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,false));
+ }
+
+ private void writeLog(AuditPayload log) {
+ this.logger.audit(log);
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/logging/ServiceLogId.java b/indexer-service-root/src/main/java/org/opendes/indexer/logging/ServiceLogId.java
new file mode 100644
index 000000000..0aec9c422
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/logging/ServiceLogId.java
@@ -0,0 +1,35 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import org.opendes.core.logging.LogId;
+
+
+public class ServiceLogId implements LogId {
+ @Override
+ public String getRequestLog() {
+ return "indexer.request";
+ }
+
+ @Override
+ public String getAuditLog() {
+ return "indexer.audit";
+ }
+
+ @Override
+ public String getAppLog() {
+ return "indexer.app";
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/middleware/IndexerFilter.java b/indexer-service-root/src/main/java/org/opendes/indexer/middleware/IndexerFilter.java
new file mode 100644
index 000000000..103120d94
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/middleware/IndexerFilter.java
@@ -0,0 +1,144 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.middleware;
+
+
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.auth.AuthorizationService;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.AuthorizationResponse;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.security.PermitAll;
+import javax.annotation.security.RolesAllowed;
+import javax.servlet.*;
+import javax.servlet.Filter;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+
+@Component
+public class IndexerFilter implements Filter {
+
+ private static final String DISABLE_AUTH_PROPERTY = "com.slb.indexer.disableAuth";
+ private static final String PATH_SWAGGER = "/swagger.json";
+ private static final String PATH_TASK_HANDLERS = "task-handlers";
+ private static final String PATH_CRON_HANDLERS = "cron-handlers";
+
+ @Context
+ private ResourceInfo resourceInfo;
+ @Autowired
+ private AuthorizationService authorizationService;
+ @Autowired
+ private JaxRsDpsLog log;
+ @Autowired
+ private IRequestInfo requestInfoProvider;
+
+ @Lazy
+ @Autowired
+ private DpsHeaders dpsHeaders;
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException {
+ boolean disableAuth = Boolean.getBoolean(DISABLE_AUTH_PROPERTY);
+ if (disableAuth) {
+ return;
+ }
+
+ HttpServletRequest httpRequest = (HttpServletRequest) request;
+ String uri = httpRequest.getRequestURI().toLowerCase();
+
+ if (uri.contains(PATH_SWAGGER)) {
+ filterChain.doFilter(request, response);
+ return;
+ }
+
+ if (httpRequest.getMethod().equals(HttpMethods.GET) && uri.contains(PATH_SWAGGER))
+ return;
+
+ if (httpRequest.getMethod().equals(HttpMethods.POST) && uri.contains(PATH_SWAGGER)) {
+ checkWorkerApiAccess(requestInfoProvider);
+ return;
+ }
+
+ if (httpRequest.getMethod().equals(HttpMethods.GET) &&uri.contains(PATH_SWAGGER)) {
+ checkCronApiAccess(requestInfoProvider);
+ return;
+ }
+
+ Method method = this.resourceInfo.getResourceMethod();
+ PermitAll permitAll = method.getAnnotation(PermitAll.class);
+ if (permitAll != null) {
+ return;
+ }
+
+ DpsHeaders headers = this.dpsHeaders;
+
+ String[] requiredRoles = getRequiredRoles(method);
+
+ checkApiAccess(requiredRoles, headers);
+ }
+
+ private void checkApiAccess(String[] requiredRoles, DpsHeaders requestHeaders) {
+ validateAccountId(requestHeaders);
+ AuthorizationResponse authorizationResponse = authorizationService.authorizeAny(requestHeaders, requiredRoles);
+ requestHeaders.put(DpsHeaders.USER_EMAIL, authorizationResponse.getUser());
+ }
+
+ private List<String> validateAccountId(DpsHeaders requestHeaders) {
+ String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId();
+ String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader);
+
+ if (Strings.isNullOrEmpty(accountHeader)) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
+ }
+
+ List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*"));
+ if (dataPartitions.isEmpty() || dataPartitions.size() > 1) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
+ }
+ return dataPartitions;
+ }
+
+ private void checkWorkerApiAccess(IRequestInfo requestInfo) {
+ if (requestInfo.isTaskQueueRequest()) return;
+ throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only");
+ }
+
+ private void checkCronApiAccess(IRequestInfo requestInfo) {
+ if (requestInfo.isCronRequest()) return;
+ throw AppException.createForbidden("invalid user agent, AppEngine Cron only");
+ }
+
+ private String[] getRequiredRoles(Method method) {
+ RolesAllowed rolesAllowed = method.getAnnotation(RolesAllowed.class);
+ return rolesAllowed.value();
+ }
+
+
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandler.java b/indexer-service-root/src/main/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandler.java
new file mode 100644
index 000000000..0b0e9015a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandler.java
@@ -0,0 +1,43 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.middleware;
+
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+
+import static org.opendes.core.util.Config.isLocalEnvironment;
+
+@Component
+public class RedirectHttpRequestsHandler implements ContainerRequestFilter {
+
+ @Autowired
+ private IRequestInfo requestInfoProvider;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+
+ if (requestInfoProvider.isTaskQueueRequest() || requestInfoProvider.isCronRequest() || isLocalEnvironment()) return;
+
+ // return 302 redirect if http connection is attempted
+ if (!requestContext.getSecurityContext().isSecure()) {
+ throw new AppException(302, "Redirect", "HTTP is not supported. Use HTTPS.");
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/ConversionStatus.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/ConversionStatus.java
new file mode 100644
index 000000000..6320a9211
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/ConversionStatus.java
@@ -0,0 +1,28 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class ConversionStatus {
+ private String id;
+ private String status;
+ private List<String> errors;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/ElasticType.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/ElasticType.java
new file mode 100644
index 000000000..c6a2102b5
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/ElasticType.java
@@ -0,0 +1,68 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import com.google.common.base.Strings;
+
+public enum ElasticType {
+
+ KEYWORD("keyword"),
+
+ TEXT("text"),
+
+ DATE("date"),
+
+ NESTED("nested"),
+
+ OBJECT("object"),
+
+ GEO_POINT("geo_point"),
+
+ GEO_SHAPE("geo_shape"),
+
+ INTEGER("integer"),
+
+ LONG("long"),
+
+ FLOAT("float"),
+
+ DOUBLE("double"),
+
+ BOOLEAN("boolean"),
+
+ UNDEFINED("undefined");
+
+ private final String value;
+
+ ElasticType(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public static ElasticType forValue(String value) {
+
+ if (Strings.isNullOrEmpty(value)) return ElasticType.UNDEFINED;
+
+ for (ElasticType type : values()) {
+ if (type.getValue().equalsIgnoreCase(value)) {
+ return type;
+ }
+ }
+ return ElasticType.UNDEFINED;
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexProgress.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexProgress.java
new file mode 100644
index 000000000..114908fd3
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexProgress.java
@@ -0,0 +1,28 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Stack;
+
+@Data
+@Builder
+public class IndexProgress {
+ private int statusCode;
+ private Stack<String> trace;
+ private String lastUpdateTime;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexSchema.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexSchema.java
new file mode 100644
index 000000000..cfa601fbf
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexSchema.java
@@ -0,0 +1,47 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+@Data
+@Builder
+public class IndexSchema {
+
+ private String kind;
+ private String type;
+ private Map<String, String> dataSchema;
+ private Map<String, Object> metaSchema;
+
+ public ArrayList<String> getSchemaKeysByValue(String value) {
+ Set<String> keys = new HashSet<>();
+ for (Map.Entry<String, String> entry : this.getDataSchema().entrySet()) {
+ if (value.equalsIgnoreCase(entry.getValue())) {
+ keys.add(entry.getKey());
+ }
+ }
+ return new ArrayList<>(keys);
+ }
+
+ public boolean isDataSchemaMissing() {
+ return dataSchema == null || dataSchema.isEmpty();
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexingStatus.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexingStatus.java
new file mode 100644
index 000000000..965c20f7b
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/IndexingStatus.java
@@ -0,0 +1,38 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+public enum IndexingStatus {
+
+ PROCESSING(0),
+
+ SUCCESS(1),
+
+ WARN(2),
+
+ SKIP(3),
+
+ FAIL(4);
+
+ private final Integer severity;
+
+ IndexingStatus(int severity) {
+ this.severity = severity;
+ }
+
+ public boolean isWorseThan(IndexingStatus other) {
+ return this.severity > other.severity;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/Legal.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/Legal.java
new file mode 100644
index 000000000..b6f6e0f03
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/Legal.java
@@ -0,0 +1,34 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Data;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+public class Legal {
+
+ @NotNull
+ @NotEmpty
+ private String[] legaltags;
+
+ @NotNull
+ @NotEmpty
+ private String[] otherRelevantDataCountries;
+
+ private String status;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/MultiFieldIndexRequest.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/MultiFieldIndexRequest.java
new file mode 100644
index 000000000..473f453b6
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/MultiFieldIndexRequest.java
@@ -0,0 +1,40 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.validator.constraints.NotBlank;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Size;
+import java.util.Set;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class MultiFieldIndexRequest {
+ @NotNull
+ @NotEmpty
+ private Set<String> indices;
+ @NotBlank
+ @Size(min=1, max=10)
+ @Builder.Default
+ private String operator = "";
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/OperationType.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/OperationType.java
new file mode 100644
index 000000000..3f9e015e1
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/OperationType.java
@@ -0,0 +1,57 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+public enum OperationType {
+ /**
+ * A post operation
+ */
+ create("create"),
+
+ /**
+ * A delete operation
+ */
+ delete("delete"),
+
+ /**
+ * A purge operation
+ */
+ purge("purge"),
+
+ /**
+ * A patch operation
+ */
+ update("update"),
+
+ /*
+ * create schema operation
+ * */
+ create_schema("create_schema"),
+
+ /*
+ * purge schema operation
+ * */
+ purge_schema("purge_schema");
+
+ private final String value;
+
+ OperationType(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/PublishMessage.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/PublishMessage.java
new file mode 100644
index 000000000..280958b2e
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/PublishMessage.java
@@ -0,0 +1,5 @@
+package org.opendes.indexer.model;
+
+public class PublishMessage {
+
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordAncestry.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordAncestry.java
new file mode 100644
index 000000000..4b8eaca32
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordAncestry.java
@@ -0,0 +1,23 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Data;
+
+@Data
+public class RecordAncestry {
+
+ private String[] parents;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIds.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIds.java
new file mode 100644
index 000000000..96d078214
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIds.java
@@ -0,0 +1,30 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RecordIds {
+ private List<String> records;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIndexerPayload.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIndexerPayload.java
new file mode 100644
index 000000000..d37583834
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordIndexerPayload.java
@@ -0,0 +1,57 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RecordIndexerPayload {
+
+ private List<IndexSchema> schemas;
+ private List<Record> records;
+
+ @Data
+ public static class Record {
+ private String id;
+ private String kind;
+ private String namespace;
+ private String type;
+ private OperationType operationType;
+ private long version;
+ private StorageAcl acl;
+ private IndexProgress indexProgress;
+ private Legal legal;
+ private RecordAncestry ancestry;
+ private Map<String, Object> data;
+ @JsonIgnore
+ private boolean schemaMissing = false;
+ @JsonIgnore
+ private boolean mappingMismatch = false;
+
+ public boolean skippedDataIndexing() {
+ return schemaMissing || mappingMismatch;
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordQueryResponse.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordQueryResponse.java
new file mode 100644
index 000000000..4d1f3dbf6
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordQueryResponse.java
@@ -0,0 +1,29 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class RecordQueryResponse {
+ private String cursor;
+ private List<String> results;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordReindexRequest.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordReindexRequest.java
new file mode 100644
index 000000000..d6e122a06
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordReindexRequest.java
@@ -0,0 +1,36 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.validator.constraints.NotBlank;
+import org.opendes.core.validation.ValidKind;
+import org.opendes.indexer.SwaggerDoc;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class RecordReindexRequest {
+
+ @NotBlank(message = SwaggerDoc.KIND_VALIDATION_CAN_NOT_BE_NULL_OR_EMPTY)
+ @ValidKind
+ private String kind;
+
+ private String cursor;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordStatus.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordStatus.java
new file mode 100644
index 000000000..3932c525a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/RecordStatus.java
@@ -0,0 +1,43 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.ToString;
+import org.springframework.beans.factory.annotation.Autowired;
+
+@Data
+@Builder
+@ToString
+public class RecordStatus {
+
+ private String id;
+ private String kind;
+ private String operationType;
+
+ @Autowired
+ private IndexingStatus status;
+
+ @Autowired
+ @ToString.Exclude private IndexProgress indexProgress;
+
+ public String getLatestTrace() {
+ if (indexProgress != null && indexProgress.getTrace() != null && indexProgress.getTrace().size() > 0) {
+ return indexProgress.getTrace().peek();
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/Records.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/Records.java
new file mode 100644
index 000000000..06779e951
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/Records.java
@@ -0,0 +1,60 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.*;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Records {
+
+ @Singular
+ private List<Entity> records;
+ private List<String> notFound;
+ @Singular
+ private List<ConversionStatus> conversionStatuses;
+
+ @Data
+ @Builder
+ @NoArgsConstructor
+ @AllArgsConstructor
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Entity {
+ private String id;
+ private long version;
+ private String kind;
+ private StorageAcl acl;
+ private Legal legal;
+ private RecordAncestry ancestry;
+ private Map<String, Object> data;
+ private List<Object> meta;
+ }
+
+ @Data
+ @Builder
+ public static class Type {
+ private String type;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/Schema.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/Schema.java
new file mode 100644
index 000000000..556b1d00e
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/Schema.java
@@ -0,0 +1,39 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Schema {
+
+ private String kind;
+ private List<Mapping> schema;
+
+ @Data
+ @Builder
+ public static class Mapping {
+ private String path;
+ private String kind;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageAcl.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageAcl.java
new file mode 100644
index 000000000..8ef0c35a7
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageAcl.java
@@ -0,0 +1,42 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import lombok.Data;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import javax.validation.constraints.NotNull;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+@Data
+public class StorageAcl {
+
+ @NotNull
+ @NotEmpty
+ private String[] viewers;
+
+ @NotNull
+ @NotEmpty
+ private String[] owners;
+
+ public static String[] flattenAcl(StorageAcl acl) {
+ Set<String> xAcl = new HashSet<>();
+ if (acl.getOwners() != null && acl.getOwners().length > 0) xAcl.addAll(Arrays.asList(acl.getOwners()));
+ if (acl.getViewers() != null && acl.getViewers().length > 0) xAcl.addAll(Arrays.asList(acl.getViewers()));
+ return xAcl.toArray(new String[xAcl.size()]);
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageType.java b/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageType.java
new file mode 100644
index 000000000..858c92409
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/model/StorageType.java
@@ -0,0 +1,52 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+public enum StorageType {
+
+ LINK("link"),
+
+ LINK_ARRAY("[]link"),
+
+ BOOLEAN("boolean"),
+
+ STRING("string"),
+
+ INT("int"),
+
+ FLOAT("float"),
+
+ DOUBLE("double"),
+
+ DOUBLE_ARRAY("[]double"),
+
+ LONG("long"),
+
+ DATETIME("datetime"),
+
+ GEO_POINT("core:dl:geopoint:1.0.0"),
+
+ GEO_SHAPE("core:dl:geoshape:1.0.0");
+
+ private final String value;
+
+ StorageType(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/publish/IPublisher.java b/indexer-service-root/src/main/java/org/opendes/indexer/publish/IPublisher.java
new file mode 100644
index 000000000..ef8709d9f
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/publish/IPublisher.java
@@ -0,0 +1,9 @@
+package org.opendes.indexer.publish;
+
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.indexer.util.JobStatus;
+
+public interface IPublisher {
+
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/AttributeParsingServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/AttributeParsingServiceImpl.java
new file mode 100644
index 000000000..e9275359e
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/AttributeParsingServiceImpl.java
@@ -0,0 +1,194 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.internal.LinkedTreeMap;
+import com.google.gson.reflect.TypeToken;
+
+import org.apache.http.HttpStatus;
+import org.opendes.core.util.Constants;
+import org.opendes.indexer.model.ElasticType;
+import org.opendes.indexer.model.IndexSchema;
+import org.opendes.indexer.model.IndexingStatus;
+import org.opendes.indexer.util.JobStatus;
+import org.opendes.indexer.util.parser.DateTimeParser;
+import org.opendes.indexer.util.parser.NumberParser;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class AttributeParsingServiceImpl implements IAttributeParsingService {
+
+ private static final String GEOJSON = "GeoJSON";
+ private static final String GEOMETRY_COLLECTION = "geometrycollection";
+ private static final String GEOMETRIES = "geometries";
+
+ @Autowired
+ private NumberParser numberParser;
+ @Autowired
+ private DateTimeParser dateTimeParser;
+ @Autowired
+ private GeometryConversionService geometryConversionService;
+ @Autowired
+ private JobStatus jobStatus;
+
+ @Override
+ public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedInteger);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedLong);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedFloat);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedDouble);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ String val = attributeVal == null ? null : String.valueOf(attributeVal);
+ dataMap.put(attributeName, Boolean.parseBoolean(val));
+ }
+
+ @Override
+ public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ String val = attributeVal == null ? null : String.valueOf(attributeVal);
+ if (Strings.isNullOrEmpty(val)) {
+ // skip indexing
+ return;
+ }
+
+ String utcDate = this.dateTimeParser.convertDateObjectToUtc(val);
+ if (Strings.isNullOrEmpty(utcDate)) {
+ String parsingError = String.format("datetime parsing error: unknown format for attribute: %s | value: %s", attributeName, attributeVal);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ } else {
+ dataMap.put(attributeName, utcDate);
+ }
+ }
+
+ @Override
+ public void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
+
+ Object attributeVal = storageRecordData.get(attributeName);
+
+ try {
+ Type type = new TypeToken<Map<String, Double>>() {}.getType();
+ Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
+
+ if (positionMap == null || positionMap.isEmpty()) return;
+
+ Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
+
+ if (position == null || position.isEmpty()) return;
+
+ dataMap.put(attributeName, position);
+
+ // check if geo shape is not there and if it is not then create it in the schema as well as create the data.
+ LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(GEOJSON);
+ if (map == null || map.isEmpty()) {
+ Map<String, Object> geometry = this.geometryConversionService.getGeopointGeoJson(positionMap);
+
+ if (geometry == null) return;
+
+ dataMap.put(DATA_GEOJSON_TAG, geometry);
+ }
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geopoint parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void tryParseGeojson(String recordId, Map<String, Object> storageRecordData, IndexSchema schemaObj, Map<String, Object> dataMap) {
+
+ final String key = "GeoJSON.features.geometry";
+ String[] strArray = key.split("\\.");
+ try {
+ // strArray[0] is GeoJSON
+ LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(strArray[0]);
+
+ if (map == null || map.isEmpty()) return;
+
+ List<Map<String, Object>> geometries = this.geometryConversionService.getGeoShape(map);
+
+ if (geometries == null || geometries.isEmpty()) return;
+
+ // Adding the point in geoshape too
+ // get the key and get the corresponding object from the storageRecord object
+ ArrayList<String> geoPointKeys = schemaObj.getSchemaKeysByValue(ElasticType.GEO_POINT.getValue());
+
+ if (geoPointKeys == null || geoPointKeys.isEmpty()) return;
+
+ LinkedTreeMap<String, Double> innerMap = (LinkedTreeMap) storageRecordData.get(geoPointKeys.get(0));
+
+ if (innerMap == null || innerMap.isEmpty()) return;
+
+ Map<String, Object> geopoint = this.geometryConversionService.getGeopointGeometry(innerMap);
+
+ if (geopoint == null || geopoint.isEmpty()) return;
+
+ geometries.add(geopoint);
+
+ Map<String, Object> outerMap = new HashMap<>();
+ outerMap.put(Constants.TYPE, GEOMETRY_COLLECTION);
+ outerMap.put(GEOMETRIES, geometries);
+
+ dataMap.put(DATA_GEOJSON_TAG, outerMap);
+ } catch (ClassCastException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, "geo-json parsing error");
+ }
+ }
+}
+
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/CronService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/CronService.java
new file mode 100644
index 000000000..e5bdd55cd
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/CronService.java
@@ -0,0 +1,24 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import java.io.IOException;
+
+public interface CronService {
+
+ boolean cleanupIndices(String regex) throws IOException;
+
+ boolean cleanupEmptyStaleIndices() throws IOException;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/CronServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/CronServiceImpl.java
new file mode 100644
index 000000000..ad0b27640
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/CronServiceImpl.java
@@ -0,0 +1,102 @@
+
+
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.IndexInfo;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+
+@Service
+public class CronServiceImpl implements CronService{
+
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+ @Autowired
+ private IRequestInfo requestInfo;
+ @Autowired
+ private IndicesService indicesService;
+ @Autowired
+ private JaxRsDpsLog log;
+
+ @Override
+ public boolean cleanupIndices(String indexPattern) throws IOException {
+ long threshHoldTime = Instant.now().minus(Config.getIndexCleanupThresholdDays(), ChronoUnit.DAYS).toEpochMilli();
+
+ try {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ final List<IndexInfo> indicesList = this.indicesService.getIndexInfo(restClient, indexPattern);
+ for (IndexInfo settings : indicesList) {
+ long indexCreateTime = Long.parseLong(settings.getCreationDate());
+ if (indexCreateTime < threshHoldTime) {
+ this.deleteIndex(restClient, settings);
+ }
+ }
+ }
+ return true;
+ } catch (ResponseException ex) {
+ throw new AppException(ex.getResponse().getStatusLine().getStatusCode(), ex.getResponse().getStatusLine().getReasonPhrase(), "Error deleting indices.", ex);
+ }
+ }
+
+ @Override
+ public boolean cleanupEmptyStaleIndices() throws IOException {
+ long threshHoldTime = Instant.now().minus(Config.getEmptyIndexCleanupThresholdDays(), ChronoUnit.DAYS).toEpochMilli();
+
+ try {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ final List<IndexInfo> indicesList = this.indicesService.getIndexInfo(restClient, null);
+ for (IndexInfo settings : indicesList) {
+ long indexCreateTime = Long.parseLong(settings.getCreationDate());
+ long documentCount = Long.parseLong(settings.getDocumentCount());
+ if (documentCount > 0) break;
+
+ if (documentCount == 0 && indexCreateTime < threshHoldTime) {
+ this.deleteIndex(restClient, settings);
+ }
+ }
+ }
+ return true;
+ } catch (ResponseException ex) {
+ throw new AppException(ex.getResponse().getStatusLine().getStatusCode(), ex.getResponse().getStatusLine().getReasonPhrase(), "Error deleting indices.", ex);
+ }
+ }
+
+ private void deleteIndex(RestHighLevelClient client, IndexInfo info) throws AppException {
+ String partitionId = this.requestInfo.getPartitionId();
+ try {
+ this.log.info(String.format("Deleting index: %s | tenant: %s | created on: %s", info.getName(), partitionId, info.getCreationDate()));
+ if (this.indicesService.deleteIndex(client, info.getName())) {
+ log.info(String.format("Deleted index: %s | tenant: %s | created on: %s", info.getName(), partitionId, info.getCreationDate()));
+ }
+ } catch (Exception ex) {
+ this.log.warning(String.format("Failed to delete index: %s | tenant: %s | created on: %s", info.getName(), partitionId, info.getCreationDate()));
+ }
+ }
+ }
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/GeometryConversionService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/GeometryConversionService.java
new file mode 100644
index 000000000..6c86e3d7f
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/GeometryConversionService.java
@@ -0,0 +1,127 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.internal.LinkedTreeMap;
+import org.opendes.core.util.Constants;
+import org.springframework.stereotype.Service;
+
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class GeometryConversionService {
+
+ private static final String POINT = "point";
+
+ private static final String GEOMETRY_COLLECTION = "geometrycollection";
+ private static final String GEOMETRIES = "geometries";
+ private static final String COORDINATES = "coordinates";
+ private static final String RADIUS = "radius";
+
+ private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat(".######");
+
+
+ @SuppressWarnings("unchecked")
+ public List<Map<String, Object>> getGeoShape(LinkedTreeMap<String, Object> map) {
+
+ String key1 = "GeoJSON.features.geometry";
+ String[] strArray = key1.split("\\.");
+
+ // strArray[1] is features
+ List<LinkedTreeMap<String, Object>> features = (ArrayList<LinkedTreeMap<String, Object>>) map.get(strArray[1]);
+ List<Map<String, Object>> geometries = new ArrayList<>();
+
+ for (LinkedTreeMap<String, Object> linkedMap : features) {
+
+ // strArray[2] is geometry
+ LinkedTreeMap<String, Object> geometry = (LinkedTreeMap) linkedMap.get(strArray[2]);
+
+ // properties
+ LinkedTreeMap<String, Object> properties = (LinkedTreeMap) linkedMap.get(Constants.PROPERTIES);
+ Map<String, Object> innerMap = new HashMap<>();
+ String type = (String) geometry.get("type");
+ innerMap.put(Constants.TYPE, type.toLowerCase());
+ innerMap.put(COORDINATES, geometry.get(COORDINATES));
+ if (properties != null && properties.size() > 0)
+ innerMap.put(Constants.PROPERTIES, properties);
+ if (geometry.get(RADIUS) != null)
+ innerMap.put(RADIUS, geometry.get(RADIUS));
+ geometries.add(innerMap);
+ }
+
+ return geometries;
+ }
+
+ public Map<String, Double> tryGetGeopoint(Map<String, Double> positionMap) {
+
+ if (positionMap == null || positionMap.size() == 0) return null;
+
+ try {
+ Map<String, Double> position = new HashMap<>();
+ double lon = new Double(DECIMAL_FORMAT.format(positionMap.get("longitude")));
+ if (lon > 180 && lon < -180)
+ throw new IllegalArgumentException("'longitude' value is out of the range [-180, 180]");
+ double lat = new Double(DECIMAL_FORMAT.format(positionMap.get("latitude")));
+ if (lat > 90 && lat < -90)
+ throw new IllegalArgumentException("'latitude' value is out of the range [-90, 90]");
+ position.put("lon", lon);
+ position.put("lat", lat);
+ return position;
+ } catch (NullPointerException | IllegalArgumentException ignored) {
+ return null;
+ }
+ }
+
+ public Map<String, Object> getGeopointGeoJson(Map<String, Double> positionMap) {
+
+ Map<String, Object> geometry = this.getGeopointGeometry(positionMap);
+ if (geometry == null) return null;
+
+ List<Map<String, Object>> geometries = new ArrayList<>();
+ geometries.add(geometry);
+
+ Map<String, Object> outerMap = new HashMap<>();
+ outerMap.put(Constants.TYPE, GEOMETRY_COLLECTION);
+ outerMap.put(GEOMETRIES, geometries);
+
+ return outerMap;
+ }
+
+ public Map<String, Object> getGeopointGeometry(Map<String, Double> positionMap) {
+
+ try {
+ double lon = new Double(DECIMAL_FORMAT.format(positionMap.get("longitude")));
+ if (lon > 180 && lon < -180)
+ throw new IllegalArgumentException("'longitude' value is out of the range [-180, 180]");
+ double lat = new Double(DECIMAL_FORMAT.format(positionMap.get("latitude")));
+ if (lat > 90 && lat < -90)
+ throw new IllegalArgumentException("'latitude' value is out of the range [-90, 90]");
+
+ Map<String, Object> inMap = new HashMap<>();
+ inMap.put(Constants.TYPE, POINT);
+ List<Double> points = new ArrayList<>();
+ points.add(lon);
+ points.add(lat);
+ inMap.put(COORDINATES, points);
+ return inMap;
+ } catch (NullPointerException | IllegalArgumentException ignored) {
+ return null;
+ }
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IAttributeParsingService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IAttributeParsingService.java
new file mode 100644
index 000000000..01819ae72
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IAttributeParsingService.java
@@ -0,0 +1,29 @@
+package org.opendes.indexer.service;
+
+import org.opendes.indexer.model.IndexSchema;
+
+import java.util.Map;
+
+public interface IAttributeParsingService {
+
+ public static final String RECORD_GEOJSON_TAG = "GeoJSON.features.geometry";
+ public static final String DATA_GEOJSON_TAG = "x-geojson";
+
+
+ void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
+
+ @SuppressWarnings("unchecked")
+ void tryParseGeojson(String recordId, Map<String, Object> storageRecordData, IndexSchema schemaObj, Map<String, Object> dataMap);
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyService.java
new file mode 100644
index 000000000..a59294dc1
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyService.java
@@ -0,0 +1,26 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.opendes.core.util.AppException;
+
+import java.io.IOException;
+
+public interface IndexCopyService {
+
+ String fetchTaskStatus(String taskId) throws AppException;
+
+ String copyIndex(String kind) throws IOException;
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyServiceImpl.java
new file mode 100644
index 000000000..925d00d4a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexCopyServiceImpl.java
@@ -0,0 +1,215 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.ClusterSettings;
+import org.opendes.core.service.ElasticSettingService;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.*;
+import org.opendes.indexer.logging.AuditLogger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+
+@Service
+public class IndexCopyServiceImpl implements IndexCopyService {
+
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+ @Autowired
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Autowired
+ private IndicesService indicesService;
+ @Autowired
+ private ElasticSettingService elasticSettingService;
+ @Autowired
+ private IndexerMappingService mappingService;
+ @Autowired
+ private HeadersInfo headersInfo;
+ @Autowired
+ private AuditLogger auditLogger;
+
+ @Override
+ public String fetchTaskStatus(String taskId) {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ Request request = new Request("GET", String.format("/_tasks/%s", taskId));
+ Response response = restClient.getLowLevelClient().performRequest(request);
+ this.auditLogger.getTaskStatus(Lists.newArrayList(taskId));
+ return EntityUtils.toString(response.getEntity());
+ } catch (IOException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", e.getMessage(), e);
+ }
+ }
+
+ /**
+ * This method is used to copy one index from common cluster to tenant cluster.
+ *
+ * @param kind request sent to ask the copying operation, includes kind and tenant
+ * @throws IOException if upstream server cannot process the request
+ */
+ @Override
+ public String copyIndex(String kind) throws IOException {
+ Preconditions.checkNotNull(kind, "kind can't be null");
+
+ String originalAccountId = this.headersInfo.getPartitionId();
+ String toBeCopiedIndex = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+
+ String typeOfToBeCopiedIndex = kind.split(":")[2];
+ if (typeOfToBeCopiedIndex == null) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Fail to find the type of the index", "Fail to find the type of the given index in common cluster.");
+ }
+
+ Map<String, Object> mappingsMap = this.getIndexMappingsFromCommonCluster(toBeCopiedIndex, typeOfToBeCopiedIndex);
+ String[] commonCluster = this.getCommonClusterInformation();
+
+ this.createIndexInTenantCluster(originalAccountId, toBeCopiedIndex, typeOfToBeCopiedIndex, mappingsMap);
+ String taskStatus = this.reindexInTenantCluster(originalAccountId, toBeCopiedIndex, commonCluster);
+ this.auditLogger.copyIndex(Lists.newArrayList(String.format("Kind:%s", kind), String.format("Task status: %s", taskStatus)));
+ return taskStatus;
+ }
+
+ /**
+ * This method is used to format the body of remote reindex request.
+ *
+ * @param host host of the remote cluster
+ * @param username username of the remote cluster
+ * @param password password of the remote cluster
+ * @param index the index to be copied
+ * @return request body in json string format
+ */
+ private String formatReindexRequestBody(String host, String username, String password, String index) {
+ Map<String, String> remoteMap = new HashMap<>();
+ remoteMap.put("host", host);
+ remoteMap.put("username", username);
+ remoteMap.put("password", password);
+
+ Map<String, Object> sourceMap = new HashMap<>();
+ sourceMap.put("index", index);
+ sourceMap.put("remote", remoteMap);
+ sourceMap.put("size", 10000);
+
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put("index", index);
+
+ Map<String, Object> map = new HashMap<>();
+ map.put("source", sourceMap);
+ map.put("dest", destMap);
+
+ Gson gson = new Gson();
+ return gson.toJson(map);
+ }
+
+ /**
+ * This method is used to extract cluster information from cluster settings.
+ *
+ * @param setting setting of the cluster, including host, port, username and password
+ * @return the host which combined the hostname, scheme and port together, username, and password
+ */
+ private String[] extractInfoFromClusterSetting(ClusterSettings setting) {
+
+ String[] clusterInfo = new String[3];
+
+ StringBuilder host = new StringBuilder("https://");
+ host.append(setting.getHost())
+ .append(":")
+ .append(setting.getPort());
+ clusterInfo[0] = host.toString();
+
+ String userAndPwd = setting.getUserNameAndPassword();
+ int indexOfColon = userAndPwd.indexOf(':');
+ clusterInfo[1] = userAndPwd.substring(0, indexOfColon);
+ clusterInfo[2] = userAndPwd.substring(indexOfColon + 1);
+
+ return clusterInfo;
+ }
+
+ /**
+ * This method is used to cast mapping to fit the parameter requirement.
+ *
+ * @param mapping input map in json string format
+ * @return output map in Map<String, Object> format
+ */
+ private Map<String, Object> castMappingsMap(String mapping, String indexType, String index) {
+ Type type = new TypeToken<Map<String, Map<String, Map<String, Map<String, Object>>>>>() {}.getType();
+ Map<String, Map<String, Map<String, Map<String, Object>>>> indexMap = new Gson().fromJson(mapping, type);
+ Map<String, Map<String, Map<String, Object>>> mappingMap = indexMap.get(index);
+ Map<String, Map<String, Object>> mappingWithoutMappings = mappingMap.get("mappings");
+
+ return mappingWithoutMappings.get(indexType);
+ }
+
+ Map<String, Object> getIndexMappingsFromCommonCluster(String toBeCopiedIndex, String typeOfToBeCopiedIndex) throws IOException {
+ this.headersInfo.getHeaders().put(DpsHeaders.ACCOUNT_ID, TenantInfo.COMMON);
+ this.headersInfo.getHeaders().put(DpsHeaders.DATA_PARTITION_ID, TenantInfo.COMMON);
+ try (RestHighLevelClient commonClient = this.elasticClientHandler.createRestClient()) {
+ String indexMapping = this.mappingService.getIndexMapping(commonClient, toBeCopiedIndex);
+ return castMappingsMap(indexMapping, typeOfToBeCopiedIndex, toBeCopiedIndex);
+ } catch (Exception e) {
+ throw new IOException("Fail to get mapping for the given index from common cluster.");
+ }
+ }
+
+ String[] getCommonClusterInformation() throws IOException {
+ this.headersInfo.getHeaders().put(DpsHeaders.ACCOUNT_ID, TenantInfo.COMMON);
+ this.headersInfo.getHeaders().put(DpsHeaders.DATA_PARTITION_ID, TenantInfo.COMMON);
+ try (RestHighLevelClient commonClient = this.elasticClientHandler.createRestClient()) {
+ String[] commonCluster = extractInfoFromClusterSetting(this.elasticSettingService.getElasticClusterInformation());
+ if (commonCluster.length != 3) {
+ throw new IOException("fail to get the information of common cluster.");
+ }
+ return commonCluster;
+ }
+ }
+
+ void createIndexInTenantCluster(String originalAccountId, String toBeCopiedIndex, String typeOfToBeCopiedIndex, Map<String, Object> mappingsMap) throws IOException {
+ this.headersInfo.getHeaders().put(DpsHeaders.ACCOUNT_ID, originalAccountId);
+ this.headersInfo.getHeaders().put(DpsHeaders.DATA_PARTITION_ID, originalAccountId);
+ try (RestHighLevelClient tenantClient = this.elasticClientHandler.createRestClient()) {
+ if (!this.indicesService.createIndex(tenantClient, toBeCopiedIndex, null, typeOfToBeCopiedIndex, mappingsMap)) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Fail to create new index", "Fail to create new corresponding new index in tenant cluster.");
+ }
+ }
+ }
+
+ String reindexInTenantCluster(String originalAccountId, String toBeCopiedIndex, String[] commonCluster) throws IOException {
+ this.headersInfo.getHeaders().put(DpsHeaders.ACCOUNT_ID, originalAccountId);
+ this.headersInfo.getHeaders().put(DpsHeaders.DATA_PARTITION_ID, originalAccountId);
+ try (RestHighLevelClient tenantClient = this.elasticClientHandler.createRestClient()) {
+ String json = formatReindexRequestBody(commonCluster[0], commonCluster[1], commonCluster[2], toBeCopiedIndex);
+ HttpEntity requestBody = new NStringEntity(json, ContentType.APPLICATION_JSON);
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(requestBody);
+ Response response = tenantClient.getLowLevelClient().performRequest(request);
+ return EntityUtils.toString(response.getEntity());
+ }
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaService.java
new file mode 100644
index 000000000..8b6eb87cb
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaService.java
@@ -0,0 +1,29 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.IndexSchema;
+import org.opendes.indexer.model.OperationType;
+
+import java.io.IOException;
+import java.util.Map;
+
+public interface IndexSchemaService {
+
+ IndexSchema getIndexerInputSchema(String kind) throws AppException;
+
+ void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaServiceImpl.java
new file mode 100644
index 000000000..f6e2152ca
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexSchemaServiceImpl.java
@@ -0,0 +1,207 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opendes.core.httpclient.RequestStatus;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.core.util.ElasticIndexNameResolver;
+import org.opendes.indexer.model.IndexSchema;
+import org.opendes.indexer.cache.SchemaCache;
+import org.opendes.indexer.model.OperationType;
+import org.opendes.indexer.model.Schema;
+import org.opendes.indexer.model.StorageType;
+import org.opendes.indexer.util.TypeMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.apache.http.HttpStatus;
+import org.springframework.stereotype.Service;
+
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+@Service
+public class IndexSchemaServiceImpl implements IndexSchemaService {
+
+ private static final String FLATTENED_SCHEMA = "_flattened";
+
+ private final Gson gson = new Gson();
+
+ @Autowired
+ private JaxRsDpsLog log;
+ @Autowired
+ private StorageService storageService;
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+ @Autowired
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Autowired
+ private IndexerMappingService mappingService;
+ @Autowired
+ private IndicesService indicesService;
+ @Autowired
+ private SchemaCache schemaCache;
+
+ public void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ schemaMsgs.entrySet().forEach(msg -> {
+ try {
+ processSchemaEvents(restClient, msg);
+ } catch (IOException | ElasticsearchStatusException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "unable to process schema update", e.getMessage());
+ }
+ });
+ }
+ }
+
+ private void processSchemaEvents(RestHighLevelClient restClient, Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
+ String kind = msg.getKey();
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+
+ boolean indexExist = this.indicesService.isIndexExist(restClient, index);
+
+ if (msg.getValue() == OperationType.create_schema) {
+ // reset cache and get new schema
+ this.invalidateCache(kind);
+ IndexSchema schemaObj = this.getIndexerInputSchema(kind);
+ if (schemaObj.isDataSchemaMissing()) {
+ log.warning(String.format("schema not found for kind: %s", kind));
+ return;
+ }
+
+ if (indexExist) {
+ try {
+ // merge the mapping
+ this.mappingService.createMapping(restClient, schemaObj, index, true);
+ } catch (AppException e) {
+ // acknowledge for TaskQueue and not retry
+ if (e.getError().getCode() == HttpStatus.SC_BAD_REQUEST) {
+ throw new AppException(RequestStatus.SCHEMA_CONFLICT, e.getError().getReason(), "error creating or merging index mapping");
+ }
+ throw e;
+ }
+ } else {
+ // create index with mapping
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schemaObj);
+ this.indicesService.createIndex(restClient, index, null, schemaObj.getType(), mapping);
+ }
+ } else if (msg.getValue() == OperationType.purge_schema) {
+ if (indexExist) {
+ // reset schema cache
+ this.invalidateCache(kind);
+ } else {
+ // log warning
+ log.warning(String.format("Kind: %s not found", kind));
+ }
+ }
+ }
+
+ @Override
+ public IndexSchema getIndexerInputSchema(String kind) throws AppException {
+
+ try {
+ String schema = this.schemaCache.get(kind);
+ if (Strings.isNullOrEmpty(schema)) {
+ // get from storage
+ schema = this.storageService.getStorageSchema(kind);
+ if (Strings.isNullOrEmpty(schema)) {
+ Schema basicSchema = Schema.builder().kind(kind).build();
+ return normalizeSchema(gson.toJson(basicSchema));
+ } else {
+ // cache the schema
+ this.schemaCache.put(kind, schema);
+ // get flatten schema and cache it
+ IndexSchema flatSchemaObj = normalizeSchema(schema);
+ if (flatSchemaObj != null) {
+ this.schemaCache.put(kind + FLATTENED_SCHEMA, gson.toJson(flatSchemaObj));
+ }
+ return flatSchemaObj;
+ }
+ } else {
+ // search flattened schema in memcache
+ String flattenedSchema = this.schemaCache.get(kind + FLATTENED_SCHEMA);
+ if (Strings.isNullOrEmpty(flattenedSchema)) {
+ Schema basicSchema = Schema.builder().kind(kind).build();
+ return normalizeSchema(gson.toJson(basicSchema));
+ }
+ return this.gson.fromJson(flattenedSchema, IndexSchema.class);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Schema parse/read error", "Error while reading schema via storage service.", e);
+ }
+ }
+
+ private void invalidateCache(String kind) {
+ String schema = this.schemaCache.get(kind);
+ if (!Strings.isNullOrEmpty(schema)) this.schemaCache.delete(kind);
+
+ String flattenSchema = this.schemaCache.get(kind + FLATTENED_SCHEMA);
+ if (!Strings.isNullOrEmpty(flattenSchema)) this.schemaCache.delete(kind + FLATTENED_SCHEMA);
+ }
+
+ private IndexSchema normalizeSchema(String schemaStr) throws AppException {
+
+ try {
+ Schema schemaObj = this.gson.fromJson(schemaStr, Schema.class);
+
+ if (schemaObj == null) return null;
+
+ Map<String, String> data = new HashMap<>();
+ Map<String, Object> meta = new HashMap<>();
+
+ if (schemaObj.getSchema() != null && !schemaObj.getSchema().isEmpty()) {
+ for (Schema.Mapping schemaInfo : schemaObj.getSchema()) {
+ String dataType = schemaInfo.getKind();
+ String elasticDataType = TypeMapper.getIndexerType(dataType);
+ if (elasticDataType == null) {
+ elasticDataType = TypeMapper.getIndexerType(StorageType.STRING.getValue());
+ }
+ data.put(schemaInfo.getPath(), elasticDataType);
+ }
+ }
+
+ // mandatory attributes
+ meta.put(RecordMetaAttribute.ID.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ID));
+ meta.put(RecordMetaAttribute.NAMESPACE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.NAMESPACE));
+ meta.put(RecordMetaAttribute.VERSION.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.VERSION));
+ meta.put(RecordMetaAttribute.KIND.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.KIND));
+ meta.put(RecordMetaAttribute.TYPE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
+ meta.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
+ meta.put(RecordMetaAttribute.X_ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+ meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
+ meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
+ meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
+
+ String kind = schemaObj.getKind();
+ String type = kind.split(":")[2];
+
+ return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
+
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Schema normalization error", "An error has occurred while normalizing the schema.", e);
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingService.java
new file mode 100644
index 000000000..6077a5478
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingService.java
@@ -0,0 +1,32 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opendes.core.service.MappingService;
+import org.opendes.indexer.model.IndexSchema;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public interface IndexerMappingService extends MappingService {
+
+ String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException;
+
+ Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema);
+
+ void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingServiceImpl.java
new file mode 100644
index 000000000..247fd37a2
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerMappingServiceImpl.java
@@ -0,0 +1,330 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+
+import com.google.gson.Gson;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.service.MappingServiceImpl;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Constants;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.core.util.Preconditions;
+import org.opendes.indexer.model.IndexSchema;
+import org.opendes.indexer.model.Records;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+
+@Service
+public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
+
+ @Autowired
+ @Lazy
+ private JaxRsDpsLog log;
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+ private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create mapping
+ */
+ public String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException {
+
+ Map<String, Object> mappingMap = this.getIndexMappingFromRecordSchema(schema);
+ String mapping = new Gson().toJson(mappingMap, Map.class);
+ this.createMappingWithJson(client, index, schema.getType(), mapping, merge);
+ return mapping;
+ }
+
+ /*
+ * Read schema mapping
+ *
+ * @param schema Index schema
+ * @param type Mapping type
+ * @return String JSON represnetation of type and elastic type
+ *
+ * sample index mapping:
+ * "properties": {
+ * all meta attributes
+ * "acl": {
+ * "properties": {
+ mapping of all roles
+ * }
+ * },
+ * "legal": {
+ * "properties": {
+ * mapping of all legal properties
+ * }
+ * }
+ * "data": {
+ * "properties": {
+ * all data-source attributes
+ * }
+ * }
+ * }
+ * */
+ public Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema) {
+
+ // entire property block
+ Map<String, Object> properties = new HashMap<>();
+
+ // meta attribute
+ Map<String, Object> metaMapping = new HashMap<>();
+ for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
+ metaMapping.put(key, entry.getValue());
+ } else {
+ metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
+ }
+ }
+
+ // data-source attributes
+ Map<String, Object> dataMapping = new HashMap<>();
+ if (schema.getDataSchema() != null) {
+ for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
+ dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
+ }
+
+ // inner properties.data.properties block
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(Constants.PROPERTIES, dataMapping);
+
+ // data & meta block
+ properties.put(Constants.DATA, dataProperties);
+ }
+ properties.putAll(metaMapping);
+
+ // entire document properties block
+ Map<String, Object> documentMapping = new HashMap<>();
+ documentMapping.put(Constants.PROPERTIES, properties);
+
+ // don't add dynamic mapping
+ documentMapping.put("dynamic", false);
+
+ return documentMapping;
+ }
+
+ @Override
+ public void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ if(!updateMappingToEnableKeywordIndexingForField(restClient,indices,fieldName)){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error updating index mapping.", String.format("Failed to get confirmation from elastic server mapping update for indices: %s", indices));
+ }
+ }
+ }
+
+ private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
+ String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
+ boolean failure = false;
+ for (String index : indicesSet) {
+ if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
+ log.info(String.format("Updated field: %s | index: %s", fieldName, index));
+ } else {
+ failure=true;
+ log.warning(String.format("Failed to update field: %s | index %s", fieldName, index));
+ }
+ }
+ return !failure;
+ }
+
+ private Map<String, Map<String, Map<String, FieldMappingMetaData>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = new HashMap<>();
+ GetFieldMappingsRequest request = new GetFieldMappingsRequest();
+ request.indices(indices);
+ request.fields(fieldNames);
+ try {
+ GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
+ if (response != null && !response.mappings().isEmpty()) {
+ final Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = response.mappings();
+ for (String index : indices) {
+ //extract mapping of each index
+ final Map<String, Map<String, FieldMappingMetaData>> indexMapping = mappings.get(index);
+ if (indexMapping != null && !indexMapping.isEmpty()) {
+ indexMappingMap.put(index, indexMapping);
+ }
+ }
+ }
+
+ return indexMappingMap;
+ } catch (ElasticsearchException e) {
+ log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
+ }
+ }
+
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, FieldMappingMetaData>> indexMapping, String fieldName) throws IOException {
+ PutMappingRequest request = new PutMappingRequest(index);
+ String type = indexMapping.keySet().iterator().next();
+ if(type.isEmpty()) {
+ log.error(String.format("Could not find type of the mappings for index: %s.", index));
+ return false;
+ }
+
+ request.type(type);
+ request.timeout(REQUEST_TIMEOUT);
+ Map<String, FieldMappingMetaData> metaData = indexMapping.get(type);
+ if(metaData==null || metaData.get("data." + fieldName)==null) {
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ FieldMappingMetaData fieldMetaData = metaData.get("data." + fieldName);
+ Map<String, Object> source = fieldMetaData.sourceAsMap();
+ if(!source.containsKey(fieldName)){
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ //Index the field with additional keyword type
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put(Constants.TYPE, "keyword");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ Map<String, Object> dataFieldMap = (Map<String, Object>) source.get(fieldName);
+ dataFieldMap.put("fields", fieldIndexTypeMap);
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(fieldName, dataFieldMap);
+ Map<String, Object> mapping = new HashMap<>();
+ mapping.put(Constants.PROPERTIES, dataProperties);
+ Map<String, Object> data = new HashMap<>();
+ data.put(Constants.DATA,mapping);
+ Map<String, Object> properties = new HashMap<>();
+ properties.put(Constants.PROPERTIES, data);
+
+ request.source(new Gson().toJson(properties), XContentType.JSON);
+
+ try {
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ boolean isIndicesUpdated = updateIndices(client, index);
+ return response.isAcknowledged() && isIndicesUpdated;
+
+ } catch (Exception e) {
+ log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
+ return false;
+ }
+ }
+
+ private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ request.setConflicts("proceed");
+ BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
+ if(!response.getBulkFailures().isEmpty()) {
+ log.error(String.format("Could not update index: %s.",index));
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create index mapping with input json
+ */
+ private void createMappingWithJson(RestHighLevelClient client, String index, String type, String mapping, boolean merge)
+ throws IOException {
+
+ boolean mappingExist = isTypeExist(client, index, type);
+ if (merge || !mappingExist) {
+ createTypeWithMappingInElasticsearch(client, index, type, mapping);
+ }
+ }
+
+ /**
+ * Check if a type already exists
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @return true if type already exists
+ * @throws IOException in case Elasticsearch responded with a status code that indicated an error
+ */
+ protected boolean isTypeExist(RestHighLevelClient client, String index, String type) throws IOException {
+
+ Request request = new Request("HEAD", "/" + index + "/_mapping/" + type);
+ Response response = client.getLowLevelClient().performRequest(request);
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @throws IOException if mapping cannot be created
+ */
+ private Boolean createTypeWithMappingInElasticsearch(RestHighLevelClient client, String index, String type, String mapping) throws IOException {
+
+ Preconditions.checkNotNull(client, "client cannot be null");
+ Preconditions.checkNotNull(index, "index cannot be null");
+ Preconditions.checkNotNull(type, "type cannot be null");
+
+ try {
+ if (mapping != null) {
+ PutMappingRequest request = new PutMappingRequest(index);
+ request.type(type);
+ request.source(mapping, XContentType.JSON);
+ request.timeout(REQUEST_TIMEOUT);
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ return response.isAcknowledged();
+ }
+ } catch (ElasticsearchException e) {
+ throw new AppException(
+ e.status().getStatus(),
+ e.getMessage(),
+ String.format("Could not create type mapping %s/%s.", index, type),
+ String.format("Failed creating mapping: %s", mapping),
+ e);
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerService.java
new file mode 100644
index 000000000..0da151906
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerService.java
@@ -0,0 +1,27 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import java.util.List;
+
+import org.opendes.core.model.RecordChangedMessages;
+import org.opendes.indexer.util.JobStatus;
+import org.opendes.indexer.util.RecordInfo;
+
+public interface IndexerService {
+
+ JobStatus processRecordChangedMessages(RecordChangedMessages recordChangedMessages, List<RecordInfo> recordInfos) throws Exception;
+
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerServiceImpl.java
new file mode 100644
index 000000000..c41f4d9c9
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/IndexerServiceImpl.java
@@ -0,0 +1,585 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.Gson;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.httpclient.RequestStatus;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.RecordChangedMessages;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Constants;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.core.util.ElasticIndexNameResolver;
+import org.opendes.indexer.logging.AuditLogger;
+import org.opendes.indexer.model.*;
+import org.opendes.indexer.publish.IPublisher;
+import org.opendes.indexer.util.IRequestInfo;
+import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+import org.opendes.indexer.util.JobStatus;
+import org.opendes.indexer.util.RecordInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.commons.beanutils.NestedNullException;
+import org.springframework.stereotype.Service;
+
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static org.opendes.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
+import static org.opendes.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
+
+@Service
+public class IndexerServiceImpl implements IndexerService {
+
+ private final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
+
+ private final Gson gson = new Gson();
+
+ @Autowired
+ private JaxRsDpsLog log;
+ @Autowired
+ private AuditLogger auditLogger;
+ @Autowired
+ private StorageService storageService;
+ @Autowired
+ private IndexSchemaService schemaService;
+ @Autowired
+ private IndicesService indicesService;
+ @Autowired
+ private IndexerMappingService mappingService;
+ @Autowired
+ private IPublisher progressPublisher;
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+ @Autowired
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Autowired
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Autowired
+ private AttributeParsingServiceImpl attributeParsingServiceImpl;
+ @Autowired
+ private IRequestInfo requestInfo;
+ @Autowired
+ private JobStatus jobStatus;
+
+ private DpsHeaders headers;
+
+ @Override
+ public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
+
+ // this should not happen
+ if (recordInfos.size() == 0) return null;
+
+ String errorMessage = "";
+ List<String> retryRecordIds = new LinkedList<>();
+
+ // get auth header
+ this.headers = this.requestInfo.getHeaders();
+
+ // initialize status for all messages.
+ this.jobStatus.initialize(recordInfos);
+
+ try {
+ // get upsert records
+ Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
+ if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
+ List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
+ retryRecordIds.addAll(upsertFailureRecordIds);
+ }
+
+ // get delete records
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
+ if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
+ List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
+ retryRecordIds.addAll(deleteFailureRecordIds);
+ }
+
+ // process schema change messages
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ this.schemaService.processSchemaMessages(schemaMsgs);
+ }
+
+ // process failed records
+ if (retryRecordIds.size() > 0) {
+ retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
+ }
+ } catch (IOException e) {
+ errorMessage = e.getMessage();
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
+ } catch (AppException e) {
+ errorMessage = e.getMessage();
+ throw e;
+ } catch (Exception e) {
+ errorMessage = "error indexing records";
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ } finally {
+ this.jobStatus.finalizeRecordStatus(errorMessage);
+ this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, jobStatus);
+ this.updateAuditLog();
+ }
+
+ return jobStatus;
+ }
+
+ private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
+
+ List<String> failedRecordIds = new LinkedList<>();
+
+ // get schema for kind
+ Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
+
+ if (schemas.isEmpty()) return new LinkedList<>();
+
+ List<String> recordIds = this.jobStatus.getIdsByIndexingStatus(IndexingStatus.PROCESSING);
+ recordIds.addAll(this.jobStatus.getIdsByIndexingStatus(IndexingStatus.SKIP));
+ recordIds.addAll(this.jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN));
+
+ if (recordIds.isEmpty()) return new LinkedList<>();
+
+ // get records via storage api
+ Records storageRecords = this.storageService.getStorageRecords(recordIds);
+
+ // map storage records to indexer payload
+ RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
+
+ // index records
+ failedRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
+
+ return failedRecordIds;
+ }
+
+ private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
+
+ Map<String, IndexSchema> schemas = new HashMap<>();
+
+ try {
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+
+ String kind = entry.getKey();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind);
+ if (schemaObj.isDataSchemaMissing()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
+ }
+
+ schemas.put(kind, schemaObj);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
+ }
+
+ return schemas;
+ }
+
+ private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
+ List<Records.Entity> storageValidRecords=records.getRecords();
+ Map<String,List<String>> conversionStatus = getConversionErrors(records.getConversionStatuses());
+ List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
+ List<IndexSchema> schemas = new ArrayList<>();
+
+ for (Records.Entity storageRecord : storageValidRecords) {
+
+ Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
+
+ String recordId=storageRecord.getId();
+ if(conversionStatus.get(recordId)!=null){
+ for(String status:conversionStatus.get(recordId)) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST,status, String.format("record-id: %s | %s", recordId, status));
+ }
+ }
+ // skip if storage returned record with same id but different kind
+ if (idOperationMap == null) {
+ String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
+ continue;
+ }
+
+ IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
+ schemas.add(schema);
+
+ // skip indexing of records if data block is empty
+ RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
+ if (document != null) {
+ indexerPayload.add(document);
+ }
+ }
+
+ log.info(String.format("valid upsert records: %s | can be indexed: %s", storageValidRecords.size(), indexerPayload.size()));
+
+ // this should only happen if storage service returned WRONG records with kind for all the records in the messages
+ if (indexerPayload.isEmpty()) {
+ throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
+ }
+
+ return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
+ }
+
+ private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
+
+ RecordIndexerPayload.Record document = null;
+
+ try {
+ Map<String, Object> storageRecordData = storageRecord.getData();
+ document = new RecordIndexerPayload.Record();
+ if (storageRecordData == null || storageRecordData.isEmpty()) {
+ String message = "empty or null data block found in the storage record";
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ document.setSchemaMissing(true);
+ } else {
+ Map<String, Object> dataMap = mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
+ if (dataMap.isEmpty()) {
+ document.setMappingMismatch(true);
+ String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ }
+ document.setData(dataMap);
+ }
+ } catch (AppException e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
+ log.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
+ log.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+
+ try {
+ // index individual parts of kind
+ String[] kindParts = storageRecord.getKind().split(":");
+
+ document.setKind(storageRecord.getKind());
+ document.setNamespace(kindParts[0] + ":" + kindParts[1]);
+ document.setType(kindParts[2]);
+ document.setId(storageRecord.getId());
+ document.setVersion(storageRecord.getVersion());
+ document.setAcl(storageRecord.getAcl());
+ document.setLegal(storageRecord.getLegal());
+ RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
+ if (recordStatus.getIndexProgress().getStatusCode() == 0) {
+ recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
+ }
+ document.setIndexProgress(recordStatus.getIndexProgress());
+ if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
+ document.setOperationType(idToOperationMap.get(storageRecord.getId()));
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
+ log.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+ return document;
+ }
+
+ private Map<String, Object> mapDataPayload(IndexSchema schemaObj, Map<String, Object> storageRecordData, String recordId) {
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ if (schemaObj.isDataSchemaMissing()) return dataMap;
+
+ // get the key and get the corresponding object from the storageRecord object
+ for (Map.Entry<String, String> entry : schemaObj.getDataSchema().entrySet()) {
+
+ String name = entry.getKey();
+
+ Object value = getPropertyValue(recordId, storageRecordData, name);
+
+ if (value == null) continue;
+
+ ElasticType elasticType = ElasticType.forValue(entry.getValue());
+
+ switch (elasticType) {
+ case KEYWORD:
+ case TEXT:
+ dataMap.put(name, value);
+ break;
+ case INTEGER:
+ this.attributeParsingServiceImpl.tryParseInteger(recordId, name, value, dataMap);
+ break;
+ case LONG:
+ this.attributeParsingServiceImpl.tryParseLong(recordId, name, value, dataMap);
+ break;
+ case FLOAT:
+ this.attributeParsingServiceImpl.tryParseFloat(recordId, name, value, dataMap);
+ break;
+ case DOUBLE:
+ this.attributeParsingServiceImpl.tryParseDouble(recordId, name, value, dataMap);
+ break;
+ case BOOLEAN:
+ this.attributeParsingServiceImpl.tryParseBoolean(recordId, name, value, dataMap);
+ break;
+ case DATE:
+ this.attributeParsingServiceImpl.tryParseDate(recordId, name, value, dataMap);
+ break;
+ case GEO_POINT:
+ this.attributeParsingServiceImpl.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
+ break;
+ case GEO_SHAPE:
+ this.attributeParsingServiceImpl.tryParseGeojson(recordId, storageRecordData, schemaObj, dataMap);
+ break;
+ case NESTED:
+ case OBJECT:
+ case UNDEFINED:
+ // don't do anything for now
+ break;
+ }
+ }
+
+ // add these once iterated over the list
+ schemaObj.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
+ schemaObj.getDataSchema().remove(RECORD_GEOJSON_TAG);
+
+ return dataMap;
+ }
+
+ private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
+ if (schemas == null || schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // process the schema
+ this.cacheOrCreateElasticMapping(schemas, restClient);
+
+ // process the records
+ return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ }
+ }
+
+ private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
+
+ for (IndexSchema schema : schemas) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+
+ // check if index exist
+ if (this.indicesService.isIndexExist(restClient, index)) continue;
+
+ // create index
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
+ if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
+ }
+ }
+ }
+
+ private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) return new LinkedList<>();
+
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (RecordIndexerPayload.Record record : records) {
+ if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
+ // it will come here when schema is missing
+ // TODO: rollback once we know what is causing the problem
+ log.warning(String.format("data not found for record: %s", record));
+ }
+
+ OperationType operation = record.getOperationType();
+ Map<String, Object> sourceMap = getSourceMap(record);
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
+
+ if (operation == OperationType.create) {
+ IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(indexRequest);
+ } else if (operation == OperationType.update) {
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(updateRequest);
+ }
+ }
+
+ return processBulkRequest(restClient, bulkRequest);
+ }
+
+ private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
+
+ String[] kindParts = record.getKey().split(":");
+ String type = kindParts[2];
+
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
+
+ for (String id : record.getValue()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ bulkRequest.add(deleteRequest);
+ }
+ }
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ return processBulkRequest(restClient, bulkRequest);
+ }
+ }
+
+ private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+
+ List<String> failureRecordIds = new LinkedList<>();
+ if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+
+ try {
+ BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+ log.info(String.format("records in bulk request: %s | acknowledged in response: %s", bulkRequest.numberOfActions(), bulkResponse.getItems().length));
+
+ // log failed bulk requests
+ ArrayList<String> bulkFailures = new ArrayList<>();
+ for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+ if (bulkItemResponse.isFailed()) {
+ BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+ bulkFailures.add(String.format("bulk status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailure().getMessage());
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
+ failureRecordIds.add(bulkItemResponse.getId());
+ }
+ } else {
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
+ }
+ }
+ if (!bulkFailures.isEmpty()) this.log.warning(bulkFailures);
+ } catch (IOException e) {
+ // throw explicit 504 for IOException
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
+ } catch (ElasticsearchStatusException e) {
+ throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
+ }
+ return failureRecordIds;
+ }
+
+ private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
+
+ Map<String, Object> indexerPayload = new HashMap<>();
+
+ // get the key and get the corresponding object from the individualRecord object
+ if (record.getData() != null) {
+ Map<String, Object> data = new HashMap<>();
+ for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
+ data.put(entry.getKey(), entry.getValue());
+ }
+ indexerPayload.put(Constants.DATA, data);
+ }
+
+ indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
+ indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
+ indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
+ indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
+ indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), StorageAcl.flattenAcl(record.getAcl()));
+ indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
+ indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
+ if (record.getAncestry() != null) {
+ indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
+ }
+ return indexerPayload;
+ }
+
+ private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
+
+ try {
+ // try getting first level property using optimized collection
+ Object propertyVal = storageRecordData.get(propertyKey);
+ if (propertyVal != null) return propertyVal;
+
+ // use apache utils to get nested property
+ return PropertyUtils.getProperty(storageRecordData, propertyKey);
+ } catch (NestedNullException ignored) {
+ // property not found in record
+ } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
+ }
+ return null;
+ }
+
+ private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
+
+ log.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
+ List<RecordInfo> retryRecordInfos = new LinkedList<>();
+ for (String recordId : failuresRecordIds) {
+ for (RecordInfo origMessage : recordInfos) {
+ if (origMessage.getId().equalsIgnoreCase(recordId)) {
+ retryRecordInfos.add(origMessage);
+ }
+ }
+ }
+
+ RecordChangedMessages newMessage = RecordChangedMessages.builder()
+ .messageId(message.getMessageId())
+ .publishTime(message.getPublishTime())
+ .data(this.gson.toJson(retryRecordInfos))
+ .attributes(message.getAttributes()).build();
+
+ String payLoad = this.gson.toJson(newMessage);
+ this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
+ }
+
+ private void updateAuditLog() {
+ logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
+ logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
+ logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
+ logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
+ }
+
+ private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
+ List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
+ if(!succeededRecords.isEmpty()) {
+ successEvent.accept(succeededRecords.stream().map(RecordStatus::toString).collect(Collectors.toList()));
+ }
+ List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
+ List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
+ failedRecords.addAll(skippedRecords);
+ if(!failedRecords.isEmpty()) {
+ failedEvent.accept(failedRecords.stream().map(RecordStatus::toString).collect(Collectors.toList()));
+ }
+ }
+
+ private Map<String,List<String>> getConversionErrors(List<ConversionStatus> conversionStatuses){
+ Map<String,List<String>> errorsByRecordId =new HashMap<>();
+ for(ConversionStatus conversionStatus:conversionStatuses){
+ if(conversionStatus.getStatus().equalsIgnoreCase("ERROR")){
+ List<String> statuses=errorsByRecordId.getOrDefault(conversionStatus.getId(),new LinkedList<>());
+ statuses.addAll(conversionStatus.getErrors());
+ errorsByRecordId.put(conversionStatus.getId(),statuses);
+ }
+ }
+ return errorsByRecordId;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexService.java
new file mode 100644
index 000000000..e45e06817
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexService.java
@@ -0,0 +1,23 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+
+import org.opendes.indexer.model.RecordReindexRequest;
+
+public interface ReindexService {
+
+ String reindexRecords(RecordReindexRequest recordReindexRequest);
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexServiceImpl.java
new file mode 100644
index 000000000..8e3643dab
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/ReindexServiceImpl.java
@@ -0,0 +1,88 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.RecordChangedMessages;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.OperationType;
+import org.opendes.indexer.model.RecordQueryResponse;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.util.IRequestInfo;
+import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+import org.opendes.indexer.util.RecordInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ReindexServiceImpl implements ReindexService {
+
+ @Autowired
+ private StorageService storageService;
+ @Autowired
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Autowired
+ private IRequestInfo requestInfo;
+ @Autowired
+ private JaxRsDpsLog log;
+
+ @Override
+ public String reindexRecords(RecordReindexRequest recordReindexRequest) {
+
+ try {
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
+
+ RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+
+ List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ Gson gson = new Gson();
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, headers);
+
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor())) {
+ String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
+ this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, headers);
+ return newPayLoad;
+ }
+
+ return recordChangedMessagePayload;
+ } else {
+ log.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
+ }
+ return null;
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageService.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageService.java
new file mode 100644
index 000000000..9ee794166
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageService.java
@@ -0,0 +1,33 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.RecordQueryResponse;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.model.Records;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.List;
+
+public interface StorageService {
+
+ Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException;
+
+ RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
+
+ String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageServiceImpl.java b/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageServiceImpl.java
new file mode 100644
index 000000000..933a3ce63
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/service/StorageServiceImpl.java
@@ -0,0 +1,136 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.httpclient.HttpResponse;
+import org.opendes.core.httpclient.RequestStatus;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.service.UrlFetchService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.indexer.model.*;
+import org.opendes.indexer.util.IRequestInfo;
+import org.opendes.indexer.util.JobStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.apache.http.HttpStatus;
+
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.opendes.core.model.SlbHeaders.SLB_FRAME_OF_REFERENCE;
+import static org.opendes.core.util.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
+
+public class StorageServiceImpl implements StorageService {
+
+ private final Gson gson = new Gson();
+
+ @Autowired
+ private UrlFetchService urlFetchService;
+ @Autowired
+ private JobStatus jobStatus;
+ @Autowired
+ private IRequestInfo requestInfo;
+ @Autowired
+ private JaxRsDpsLog log;
+
+ @Override
+ public Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
+ List<Records.Entity> valid = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ List<ConversionStatus> conversionStatuses = new ArrayList<>();
+
+ List<List<String>> batch = Lists.partition(ids, Config.getStorageRecordsBatchSize());
+ for (List<String> recordsBatch : batch) {
+ Records storageOut = this.getRecords(recordsBatch);
+ valid.addAll(storageOut.getRecords());
+ notFound.addAll(storageOut.getNotFound());
+ conversionStatuses.addAll(storageOut.getConversionStatuses());
+ }
+ return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).build();
+ }
+
+ private Records getRecords(List<String> ids) throws URISyntaxException {
+ // e.g. {"records":["test:10"]}
+ String body = this.gson.toJson(RecordIds.builder().records(ids).build());
+
+// Map<String, String> headers = this.requestInfo.getHeadersMap();
+ DpsHeaders headers = this.requestInfo.getHeaders();
+ headers.put(SLB_FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
+ HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.POST, Config.getStorageQueryRecordFoRConversionHostUrl(), headers, null, body);
+ String dataFromStorage = response.getBody();
+ if (Strings.isNullOrEmpty(dataFromStorage)) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service cannot locate records");
+ }
+
+ Type recordsListType = new TypeToken<Records>() {}.getType();
+ Records records = this.gson.fromJson(dataFromStorage, recordsListType);
+
+ // update status for invalid records from storage
+ if (records.getNotFound() != null && !records.getNotFound().isEmpty()) {
+ this.jobStatus.addOrUpdateRecordStatus(records.getNotFound(), IndexingStatus.FAIL, RequestStatus.INVALID_RECORD, "invalid storage records", String.format("invalid records: %s", String.join(",", records.getNotFound())));
+ }
+
+ // don't proceed if there is nothing to process
+ List<Records.Entity> validRecords = records.getRecords();
+ if (validRecords == null || validRecords.isEmpty()) {
+ if (response.isSuccessCode()) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Storage service returned retry or invalid records");
+ }
+
+ // TODO: returned actual code from storage service
+ log.warning(String.format("unable to proceed, valid storage record not found. | upstream response code: %s | record ids: %s", response.getResponseCode(), String.join(" | ", ids)));
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service cannot locate valid records");
+ }
+
+ return records;
+ }
+
+ @Override
+ public RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException {
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put(RecordMetaAttribute.KIND.getValue(), request.getKind());
+ queryParams.put("limit", String.valueOf(Config.getStorageRecordsBatchSize()));
+ if (!Strings.isNullOrEmpty(request.getCursor())) {
+ queryParams.put("cursor", request.getCursor());
+ }
+
+ if(requestInfo == null)
+ throw new AppException(HttpStatus.SC_NO_CONTENT, "Invalid header", "header can't be null");
+
+ HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.GET, Config.getStorageQueryRecordHostUrl(), this.requestInfo.getHeaders(), queryParams, null);
+ return this.gson.fromJson(response.getBody(), RecordQueryResponse.class);
+ }
+
+ @Override
+ public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
+ String url = String.format("%s/%s", Config.getStorageSchemaHostUrl(), URLEncoder.encode(kind, "UTF-8"));
+ HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.GET, url, this.requestInfo.getHeaders(), null, null);
+ if (response.getResponseCode() != HttpStatus.SC_OK) return null;
+ return response.getBody();
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/swagger/HomeController.java b/indexer-service-root/src/main/java/org/opendes/indexer/swagger/HomeController.java
new file mode 100644
index 000000000..7120c9c11
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/swagger/HomeController.java
@@ -0,0 +1,13 @@
+package org.opendes.indexer.swagger;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+@Controller
+public class HomeController {
+ @RequestMapping(value = "/swagger")
+ public String swagger() {
+ System.out.println("swagger-ui.html");
+ return "redirect:swagger-ui.html";
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/swagger/SwaggerDocumentationConfig.java b/indexer-service-root/src/main/java/org/opendes/indexer/swagger/SwaggerDocumentationConfig.java
new file mode 100644
index 000000000..5ed67e6b3
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/swagger/SwaggerDocumentationConfig.java
@@ -0,0 +1,76 @@
+package org.opendes.indexer.swagger;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ParameterBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.schema.ModelRef;
+import springfox.documentation.service.ApiKey;
+import springfox.documentation.service.AuthorizationScope;
+import springfox.documentation.service.SecurityReference;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spi.service.contexts.SecurityContext;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+import springfox.documentation.service.Parameter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerDocumentationConfig {
+ public static final String AUTHORIZATION_HEADER = "Authorization";
+ public static final String DEFAULT_INCLUDE_PATTERN = "/.*";
+
+ @Bean
+ public Docket api() {
+ ParameterBuilder builder = new ParameterBuilder();
+ List<Parameter> parameters = new ArrayList<>();
+ builder.name("slb-data-partition-id")
+ .description("tenant")
+ .defaultValue("common")
+ .modelRef(new ModelRef("string"))
+ .parameterType("header")
+ .required(true)
+ .build();
+ parameters.add(builder.build());
+ builder.name("slb-frame-of-reference")
+ .description("reference")
+ .defaultValue("none")
+ .modelRef(new ModelRef("string"))
+ .parameterType("header")
+ .required(true)
+ .build();
+ parameters.add(builder.build());
+ return new Docket(DocumentationType.SWAGGER_2)
+ .globalOperationParameters(parameters)
+ .select()
+ .apis(RequestHandlerSelectors.basePackage("com.slb.storage.api"))
+ .build()
+ .securityContexts(Collections.singletonList(securityContext()))
+ .securitySchemes(Collections.singletonList(apiKey()));
+ }
+
+ private ApiKey apiKey() {
+ return new ApiKey("JWT", AUTHORIZATION_HEADER, "header");
+ }
+
+ private SecurityContext securityContext() {
+ return SecurityContext.builder()
+ .securityReferences(defaultAuth())
+ .forPaths(PathSelectors.regex(DEFAULT_INCLUDE_PATTERN))
+ .build();
+ }
+
+ List<SecurityReference> defaultAuth() {
+ AuthorizationScope authorizationScope
+ = new AuthorizationScope("global", "accessEverything");
+ AuthorizationScope[] authorizationScopes =
+ new AuthorizationScope[]{authorizationScope};
+ return Collections.singletonList(
+ new SecurityReference("JWT", authorizationScopes));
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/IRequestInfo.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/IRequestInfo.java
new file mode 100644
index 000000000..690b53930
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/IRequestInfo.java
@@ -0,0 +1,22 @@
+package org.opendes.indexer.util;
+
+import org.opendes.client.api.DpsHeaders;
+
+import java.util.Map;
+
+public interface IRequestInfo {
+ public DpsHeaders getHeaders();
+
+ public String getPartitionId();
+
+ public Map<String, String> getHeadersMap();
+
+ public Map<String, String> getHeadersMapWithDwdAuthZ();
+
+ public DpsHeaders getHeadersWithDwdAuthZ();
+
+
+ public boolean isCronRequest();
+
+ public boolean isTaskQueueRequest();
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/IndexerQueueTaskBuilder.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/IndexerQueueTaskBuilder.java
new file mode 100644
index 000000000..b38ef5f76
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/IndexerQueueTaskBuilder.java
@@ -0,0 +1,66 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.gson.Gson;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.httpclient.HttpResponse;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.CloudTaskRequest;
+import org.opendes.core.service.UrlFetchService;
+import org.opendes.core.util.Config;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import static org.opendes.core.util.Constants.REINDEX_RELATIVE_URL;
+import static org.opendes.core.util.Constants.WORKER_RELATIVE_URL;
+
+@Component
+public class IndexerQueueTaskBuilder {
+
+ @Autowired
+ private UrlFetchService urlFetchService;
+ @Autowired
+ private JaxRsDpsLog log;
+
+ public void createWorkerTask(String payload, DpsHeaders headers) {
+ createTask(WORKER_RELATIVE_URL, payload, headers);
+ }
+
+ public void createReIndexTask(String payload,DpsHeaders headers) {
+ createTask(REINDEX_RELATIVE_URL, payload, headers);
+ }
+
+ private void createTask(String url, String payload, DpsHeaders headers) {
+
+ CloudTaskRequest cloudTaskRequest = CloudTaskRequest.builder().message(payload).url(url).build();
+
+ try {
+ HttpResponse response = this.urlFetchService.sendRequest(
+ HttpMethods.POST,
+ Config.getIndexerQueueHost(),
+ headers,
+ null,
+ new Gson().toJson(cloudTaskRequest));
+ this.log.info(String.format("task enqueuing response: %s", response.getResponseCode()));
+ } catch (URISyntaxException e) {
+ this.log.warning(String.format("error enqueuing task message: %s | url: %s | task payload: %s", e.getMessage(), url, payload));
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/JobStatus.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/JobStatus.java
new file mode 100644
index 000000000..e30a01246
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/JobStatus.java
@@ -0,0 +1,132 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import com.google.common.base.Strings;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.indexer.model.*;
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Data
+public class JobStatus {
+
+ @Autowired
+ private JaxRsDpsLog log;
+
+ @Autowired
+ private List<RecordStatus> statusesList;
+
+ @Autowired
+ private List<String> debugInfos;
+
+ public void initialize(List<RecordInfo> recordInfos) {
+
+ if (recordInfos == null || recordInfos.isEmpty()) return;
+
+ List<RecordStatus> statuses = recordInfos.stream().map(msg -> RecordStatus.builder()
+ .id(msg.getId())
+ .kind(msg.getKind())
+ .operationType(msg.getOp())
+ .status(IndexingStatus.PROCESSING)
+ .indexProgress(IndexProgress.builder().trace(new Stack<>()).lastUpdateTime(Instant.now().toString()).build())
+ .build()).collect(Collectors.toList());
+
+ this.statusesList.addAll(statuses);
+ }
+
+ public void addOrUpdateRecordStatus(Collection<String> ids, IndexingStatus status, int statusCode, String message, String debugInfo) {
+
+ this.debugInfos.add(debugInfo);
+ addOrUpdateRecordStatus(ids, status, statusCode, message);
+ }
+
+ public void addOrUpdateRecordStatus(String id, IndexingStatus status, int statusCode, String message, String debugInfo) {
+
+ this.debugInfos.add(debugInfo);
+ addOrUpdateRecordStatus(id, status, statusCode, message);
+ }
+
+ public void addOrUpdateRecordStatus(Collection<String> ids, IndexingStatus status, int statusCode, String message) {
+
+ if (ids == null || ids.isEmpty()) return;
+ ids.forEach(id -> addOrUpdateRecordStatus(id, status, statusCode, message));
+ }
+
+ public void addOrUpdateRecordStatus(String id, IndexingStatus status, int statusCode, String message) {
+ Optional<RecordStatus> queryResult = this.statusesList.stream().filter(s -> s.getId().equalsIgnoreCase(id)).findFirst();
+ if (queryResult.isPresent()) {
+ RecordStatus s = queryResult.get();
+ IndexProgress indexProgress = s.getIndexProgress();
+ indexProgress.setStatusCode(statusCode);
+ indexProgress.setLastUpdateTime(Instant.now().toString());
+ if (!Strings.isNullOrEmpty(message)) {
+ indexProgress.getTrace().add(message);
+ }
+ if (status.isWorseThan(s.getStatus())) {
+ s.setStatus(status);
+ }
+ s.setIndexProgress(indexProgress);
+ } else {
+ IndexProgress indexProgress = IndexProgress.builder()
+ .trace(new Stack<>())
+ .lastUpdateTime(Instant.now().toString()).build();
+ indexProgress.getTrace().add(message);
+ this.statusesList.add(RecordStatus.builder().id(id).status(status).indexProgress(indexProgress).build());
+ }
+ }
+
+ public List<String> getIdsByIndexingStatus(IndexingStatus indexingStatus) {
+
+ return this.statusesList.stream().filter(s -> s.getStatus() == indexingStatus).map(RecordStatus::getId)
+ .collect(Collectors.toList());
+ }
+
+ public String getRecordKindById(String id) {
+ Optional<RecordStatus> optionalRecordStatus = this.statusesList.stream().filter(s -> s.getId()
+ .equalsIgnoreCase(id)).findFirst();
+ RecordStatus status = optionalRecordStatus.orElse(null);
+ return status != null ? status.getKind() : null;
+ }
+
+ public RecordStatus getJobStatusByRecordId(String id) {
+ Optional<RecordStatus> optionalRecordStatus = this.statusesList.stream().filter(s -> s.getId()
+ .equalsIgnoreCase(id)).findFirst();
+ return optionalRecordStatus.orElse(null);
+ }
+
+ public List<RecordStatus> getRecordStatuses(IndexingStatus indexingStatus, OperationType operationType) {
+ return this.statusesList.stream().filter(
+ s -> s.getStatus() == indexingStatus && s.getOperationType().equalsIgnoreCase(operationType.getValue())).collect(Collectors.toList());
+ }
+
+ /*
+ * mark all the records as FAIL if for some reason they were not processed
+ * */
+ public void finalizeRecordStatus(String errorMessage) {
+ statusesList.stream().filter(recordStatus -> recordStatus.getStatus() == IndexingStatus.PROCESSING).forEach
+ (recordStatus -> {
+ recordStatus.setStatus(IndexingStatus.FAIL);
+ recordStatus.getIndexProgress().getTrace().add(errorMessage);
+ });
+
+ // dump all debug-info
+ this.log.warning(this.debugInfos);
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/RecordInfo.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/RecordInfo.java
new file mode 100644
index 000000000..11b3d7750
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/RecordInfo.java
@@ -0,0 +1,98 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.http.HttpStatus;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.OperationType;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class RecordInfo {
+
+ private static final long serialVersionUID = 1L;
+
+ private String id;
+ private String kind;
+ private String op;
+
+ public static Map<String, Map<String, OperationType>> getUpsertRecordIds(List<RecordInfo> msgs) throws AppException {
+
+ Map<String, Map<String, OperationType>> kindRecordOpMap = new HashMap<>();
+
+ try {
+ for (RecordInfo msg : msgs) {
+ OperationType op = OperationType.valueOf(msg.getOp());
+ if (op == OperationType.create || op == OperationType.update) {
+ Map<String, OperationType> idOperationMap = kindRecordOpMap.containsKey(msg.getKind()) ? kindRecordOpMap.get(msg.getKind()) : new HashMap<>();
+ idOperationMap.put(msg.getId(), OperationType.valueOf(msg.getOp()));
+ kindRecordOpMap.put(msg.getKind(), idOperationMap);
+ }
+ }
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing upsert records in request payload.", e);
+ }
+ return kindRecordOpMap;
+ }
+
+ public static Map<String, List<String>> getDeleteRecordIds(List<RecordInfo> msgs) {
+
+ Map<String, List<String>> deleteRecordMap = new HashMap<>();
+
+ try {
+ for (RecordInfo msg : msgs) {
+ OperationType op = OperationType.valueOf(msg.getOp());
+ if (op == OperationType.purge || op == OperationType.delete) {
+ String kind = msg.getKind();
+ if (!deleteRecordMap.containsKey(kind)) {
+ deleteRecordMap.put(kind, new ArrayList<>());
+ }
+ deleteRecordMap.get(kind).add(msg.getId());
+ }
+ }
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing delete records in request payload.", e);
+ }
+ return deleteRecordMap;
+ }
+
+ public static Map<String, OperationType> getSchemaMsgs(List<RecordInfo> msgs) {
+
+ Map<String, OperationType> schemaOperations = new HashMap<>();
+
+ try {
+ for (RecordInfo msg : msgs) {
+ OperationType op = OperationType.valueOf(msg.getOp());
+ if (op == OperationType.create_schema || op == OperationType.purge_schema) {
+ schemaOperations.put(msg.getKind(), op);
+ }
+ }
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing schema updates in request payload.", e);
+ }
+ return schemaOperations;
+ }
+}
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/TypeMapper.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/TypeMapper.java
new file mode 100644
index 000000000..12a917a61
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/TypeMapper.java
@@ -0,0 +1,113 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import org.opendes.core.model.AclRole;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.util.Constants;
+import org.opendes.indexer.model.ElasticType;
+import org.opendes.indexer.model.Records;
+import org.opendes.indexer.model.StorageType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TypeMapper {
+
+ private static final Map<String, String> storageToIndexerType = new HashMap<>();
+
+ private static final Map<String, Object> metaAttributeIndexerType = new HashMap<>();
+
+ static {
+
+ metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.TYPE.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.ID.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.NAMESPACE.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.VERSION.getValue(), ElasticType.LONG.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.X_ACL.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.ACL.getValue(), getAclIndexerMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
+
+ storageToIndexerType.put(StorageType.LINK.getValue(), ElasticType.KEYWORD.getValue());
+ storageToIndexerType.put(StorageType.LINK_ARRAY.getValue(), ElasticType.KEYWORD.getValue());
+ storageToIndexerType.put(StorageType.BOOLEAN.getValue(), ElasticType.BOOLEAN.getValue());
+ storageToIndexerType.put(StorageType.STRING.getValue(), ElasticType.TEXT.getValue());
+ storageToIndexerType.put(StorageType.INT.getValue(), ElasticType.INTEGER.getValue());
+ storageToIndexerType.put(StorageType.FLOAT.getValue(), ElasticType.FLOAT.getValue());
+ storageToIndexerType.put(StorageType.DOUBLE.getValue(), ElasticType.DOUBLE.getValue());
+ storageToIndexerType.put(StorageType.DOUBLE_ARRAY.getValue(), ElasticType.DOUBLE.getValue());
+ storageToIndexerType.put(StorageType.LONG.getValue(), ElasticType.LONG.getValue());
+ storageToIndexerType.put(StorageType.DATETIME.getValue(), ElasticType.DATE.getValue());
+ storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
+ storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
+ }
+
+
+ public static String getIndexerType(String storageType) {
+ return storageToIndexerType.getOrDefault(storageType, null);
+ }
+
+ public static Object getIndexerType(RecordMetaAttribute attribute) {
+ return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
+ }
+
+ private static Object getAclIndexerMapping() {
+ Map<String, Object> aclRoleMapping = new HashMap<>();
+ aclRoleMapping.put(AclRole.VIEWERS.getValue() , Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ aclRoleMapping.put(AclRole.OWNERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+
+ Map<String, Object> aclProperties = new HashMap<>();
+ aclProperties.put(Constants.PROPERTIES, aclRoleMapping);
+
+ return aclProperties;
+ }
+
+ private static Object getLegalIndexerMapping() {
+ Map<String, Object> legalComplianceMapping = new HashMap<>();
+ legalComplianceMapping.put("legaltags", Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ legalComplianceMapping.put("otherRelevantDataCountries", Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ legalComplianceMapping.put("status", Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+
+ Map<String, Object> legalProperties = new HashMap<>();
+ legalProperties.put(Constants.PROPERTIES, legalComplianceMapping);
+
+ return legalProperties;
+ }
+
+ private static Object getAncestryIndexerMapping() {
+ Map<String, Object> ancestryMapping = new HashMap<>();
+ ancestryMapping.put("parents", Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+
+ Map<String, Object> ancestryProperties = new HashMap<>();
+ ancestryProperties.put(Constants.PROPERTIES, ancestryMapping);
+
+ return ancestryProperties;
+ }
+
+ private static Object getIndexStatusMapping() {
+ Map<String, Object> indexStatusMapping = new HashMap<>();
+ indexStatusMapping.put("statusCode", Records.Type.builder().type(ElasticType.INTEGER.getValue()).build());
+ indexStatusMapping.put("trace", Records.Type.builder().type(ElasticType.TEXT.getValue()).build());
+ indexStatusMapping.put("lastUpdateTime", Records.Type.builder().type(ElasticType.DATE.getValue()).build());
+
+ Map<String, Object> indexStatusProperties = new HashMap<>();
+ indexStatusProperties.put(Constants.PROPERTIES, indexStatusMapping);
+
+ return indexStatusProperties;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/DateTimeParser.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/DateTimeParser.java
new file mode 100644
index 000000000..38c0e88e1
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/DateTimeParser.java
@@ -0,0 +1,84 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util.parser;
+
+import com.google.common.base.Strings;
+import org.springframework.stereotype.Component;
+
+import java.time.DateTimeException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoField;
+import java.util.ArrayList;
+import java.util.List;
+
+@Component
+public class DateTimeParser {
+
+ private final static List<DateTimeFormatter> KNOWN_FORMATTERS = new ArrayList<>();
+ static {
+ KNOWN_FORMATTERS.add(new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.YEAR, 4)
+ .appendPattern("[-]MM[-]dd['T'][ ][HH][:][mm][:][ss]")
+ .optionalStart().appendFraction(ChronoField.MICRO_OF_SECOND, 0, 7, true).optionalEnd()
+ .optionalStart().appendPattern("[,S][XXX]").optionalEnd()
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .parseDefaulting(ChronoField.OFFSET_SECONDS, 0)
+ .toFormatter());
+ KNOWN_FORMATTERS.add(new DateTimeFormatterBuilder()
+ .appendPattern("EEE MMM dd HH:mm:ss zzz yyyy")
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .parseDefaulting(ChronoField.OFFSET_SECONDS, 0)
+ .toFormatter());
+ }
+
+ private final DateTimeFormatter UTC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
+ // optional offset - prints +0000 when it's zero (instead of Z)
+ .optionalStart().appendOffset("+HHMM", "+0000").optionalStart()
+ .optionalStart()
+ // optional zone id (so it parses "Z")
+ .appendZoneId()
+ // add default value for offset seconds when field is not present
+ .parseDefaulting(ChronoField.OFFSET_SECONDS, 0).optionalEnd()
+ .toFormatter();
+
+ public String convertDateObjectToUtc(String candidate) {
+ if (Strings.isNullOrEmpty(candidate)) return null;
+
+ OffsetDateTime parsedDateTime = null;
+ for(DateTimeFormatter formatter: KNOWN_FORMATTERS) {
+ try {
+ parsedDateTime = OffsetDateTime.parse(candidate, formatter);
+ } catch (DateTimeParseException ignored) {
+ continue;
+ }
+ break;
+ }
+ if (parsedDateTime != null) {
+ try {
+ return parsedDateTime.format(UTC_FORMATTER);
+ } catch (DateTimeException ignored) {
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/NumberParser.java b/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/NumberParser.java
new file mode 100644
index 000000000..2dece1d6a
--- /dev/null
+++ b/indexer-service-root/src/main/java/org/opendes/indexer/util/parser/NumberParser.java
@@ -0,0 +1,147 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util.parser;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Numbers;
+import org.springframework.stereotype.Component;
+
+@Component
+public class NumberParser {
+
+ public int parseInteger(String attributeName, Object attributeVal) {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+
+ // elastic allows empty value for numeric types
+ if (Strings.isNullOrEmpty(value)) {
+ return 0;
+ }
+
+ if (!NumberUtils.isCreatable(value)) {
+ throw new IllegalArgumentException(String.format("number parsing error: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ double doubleValue = objectToDouble(value);
+ if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException(String.format("number parsing error, integer out of range: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ if (attributeVal instanceof Number) {
+ return ((Number) attributeVal).intValue();
+ }
+
+ return (int) doubleValue;
+ }
+
+ public long parseLong(String attributeName, Object attributeVal) {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+
+ // elastic allows empty value for numeric types
+ if (Strings.isNullOrEmpty(value)) {
+ return 0L;
+ }
+
+ if (!NumberUtils.isCreatable(value)) {
+ throw new IllegalArgumentException(String.format("number parsing error: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ if (attributeVal instanceof Long) {
+ return (Long) attributeVal;
+ }
+
+ try {
+ double doubleValue = objectToDouble(value);
+ // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
+ // be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
+ if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
+ throw new IllegalArgumentException(String.format("number parsing error, long out of range: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+ } catch (NumberFormatException e) {
+ // Numbers.toLong will try with BigDecimal
+ }
+
+ // longs need special handling so we don't lose precision while parsing
+ String stringValue = (attributeVal instanceof BytesRef) ? ((BytesRef) attributeVal).utf8ToString() : value.toString();
+ return Numbers.toLong(stringValue, false);
+ }
+
+ public float parseFloat(String attributeName, Object attributeVal) {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+
+ // elastic allows empty value for numeric types
+ if (Strings.isNullOrEmpty(value)) {
+ return 0.0f;
+ }
+
+ if (!NumberUtils.isCreatable(value)) {
+ throw new IllegalArgumentException(String.format("number parsing error: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ final float result;
+ if (attributeVal instanceof Number) {
+ result = ((Number) attributeVal).floatValue();
+ } else {
+ if (attributeVal instanceof BytesRef) {
+ attributeVal = ((BytesRef) attributeVal).utf8ToString();
+ }
+ result = Float.parseFloat(attributeVal.toString());
+ }
+
+ if (!Float.isFinite(result)) {
+ throw new IllegalArgumentException(String.format("number parsing error, float only supports finite values: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ return result;
+ }
+
+ public double parseDouble(String attributeName, Object attributeVal) {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+
+ // elastic allows empty value for numeric types
+ if (Strings.isNullOrEmpty(value)) {
+ return 0.0d;
+ }
+
+ if (!NumberUtils.isCreatable(value)) {
+ throw new IllegalArgumentException(String.format("number parsing error: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ double doubleValue = objectToDouble(value);
+ if (!Double.isFinite(doubleValue)) {
+ throw new IllegalArgumentException(String.format("number parsing error, double only supports finite values: attribute: %s | value: %s", attributeName, attributeVal));
+ }
+
+ return doubleValue;
+ }
+
+ /**
+ * Converts an Object to a double by checking it against known types first
+ */
+ private static double objectToDouble(Object value) {
+ double doubleValue;
+
+ if (value instanceof Number) {
+ doubleValue = ((Number) value).doubleValue();
+ } else if (value instanceof BytesRef) {
+ doubleValue = Double.parseDouble(((BytesRef) value).utf8ToString());
+ } else {
+ doubleValue = Double.parseDouble(value.toString());
+ }
+
+ return doubleValue;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/api/HealthCheckApiTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/api/HealthCheckApiTest.java
new file mode 100644
index 000000000..5d28e5631
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/api/HealthCheckApiTest.java
@@ -0,0 +1,46 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.api;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static junit.framework.TestCase.assertEquals;
+
+@RunWith(SpringRunner.class)
+public class HealthCheckApiTest {
+
+ @InjectMocks
+ private HealthCheckApi sut;
+
+ @Before
+ public void setup() {
+ this.sut = new HealthCheckApi();
+ }
+
+ @Test
+ public void should_returnHttp200_when_checkLiveness() {
+ assertEquals(HttpStatus.OK.value(), this.sut.livenessCheck().getStatusCodeValue());
+ }
+
+ @Test
+ public void should_returnHttp200_when_checkReadiness() {
+ assertEquals(HttpStatus.OK.value(), this.sut.readinessCheck().getStatusCodeValue());
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/api/RecordIndexerApiTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/api/RecordIndexerApiTest.java
new file mode 100644
index 000000000..a19ef8bb1
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/api/RecordIndexerApiTest.java
@@ -0,0 +1,121 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.api;
+
+import com.google.gson.Gson;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.model.RecordChangedMessages;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.HeadersUtil;
+import org.opendes.indexer.service.IndexerService;
+import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({HeadersUtil.class, IndexerQueueTaskBuilder.class, DpsHeaders.class, Config.class})
+public class RecordIndexerApiTest {
+
+ private final String messageValid = "{\"data\":\"[{\\\"id\\\":\\\"tenant1:welldb:wellbore-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\\\",\\\"kind\\\":\\\"tenant1:welldb:wellbore:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-account-id\":\"tenant1\",\"slb-correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
+ private final String messageEmpty = "{}";
+ private final String messageWithEmptyData = "{\"data\":\"[]\",\"attributes\":{\"slb-account-id\":\"tenant1\",\"slb-correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
+ private final String messageWithIncorrectJsonFormat = "{\"data\":\"[{}}]\",\"attributes\":{\"slb-account-id\":\"tenant1\",\"slb-correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
+
+ private final String ACCOUNT_ID = "any-account";
+
+ @InjectMocks
+ private RecordIndexerApi sut;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IndexerService indexService;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+// mockStatic(HeadersUtil.class);
+// mockStatic(IndexerQueueTaskBuilder.class);
+// mockStatic(DpsHeaders.class);
+// mockStatic(Config.class);
+
+ DpsHeaders dpsHeaders = new DpsHeaders();
+ dpsHeaders.put(DpsHeaders.ACCOUNT_ID, this.ACCOUNT_ID);
+ when(DpsHeaders.createFromEntrySet(any())).thenReturn(dpsHeaders);
+
+ when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+ }
+
+ @Test
+ public void should_return200_given_validMessage_indexWorkerTest() throws Exception {
+ should_return200_indexerWorkerTest(messageValid);
+ }
+
+ @Test
+ public void should_return200_given_emptyData_indexWorkerTest() throws Exception {
+ should_return200_indexerWorkerTest(messageWithEmptyData);
+ }
+
+ @Test
+ public void should_return400_given_emptyMessage_indexWorkerTest() {
+ should_return400_indexerWorkerTest(messageEmpty, "Required header: 'slb-data-partition-id' not found");
+ }
+
+ @Test
+ public void should_return400_given_incorrectJsonFormatMessage_indexWorkerTest() {
+ should_return400_indexerWorkerTest(messageWithIncorrectJsonFormat, "Unable to parse request payload.");
+ }
+
+ private void should_return200_indexerWorkerTest(String message) throws Exception {
+ ResponseEntity response = this.sut.indexWorker(createRecordChangedMessage(message));
+ Assert.assertEquals(HttpStatus.OK, response.getStatusCodeValue());
+ }
+
+ private void should_return400_indexerWorkerTest(String message, String errorMessage) {
+ try {
+ this.sut.indexWorker(createRecordChangedMessage(message));
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.BAD_REQUEST, e.getError().getCode());
+ Assert.assertEquals(errorMessage, e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ private RecordChangedMessages createRecordChangedMessage(String message) {
+ return (new Gson()).fromJson(message, RecordChangedMessages.class);
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/api/ReindexApiTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/api/ReindexApiTest.java
new file mode 100644
index 000000000..18a24904b
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/api/ReindexApiTest.java
@@ -0,0 +1,74 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.api;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.logging.AuditLogger;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.service.ReindexService;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class ReindexApiTest {
+
+ private RecordReindexRequest recordReindexRequest;
+
+ @Mock
+ private ReindexService reIndexService;
+ @Mock
+ private AuditLogger auditLogger;
+ @InjectMocks
+ private ReindexApi sut;
+
+ @Before
+ public void setup() {
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+ }
+
+ @Test
+ public void should_return200_when_valid_kind_provided() {
+ when(this.reIndexService.reindexRecords(recordReindexRequest)).thenReturn("something");
+
+ ResponseEntity response = sut.reindex(recordReindexRequest);
+
+ assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() {
+ when(this.reIndexService.reindexRecords(any())).thenThrow(new AppException(500, "", ""));
+
+ sut.reindex(recordReindexRequest);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() {
+ when(this.reIndexService.reindexRecords(any())).thenThrow(new NullPointerException(""));
+
+ sut.reindex(recordReindexRequest);
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditEventsTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditEventsTest.java
new file mode 100644
index 000000000..4eff8a6c8
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditEventsTest.java
@@ -0,0 +1,230 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opendes.client.logging.payload.AuditAction;
+import org.opendes.client.logging.payload.AuditStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(SpringRunner.class)
+public class AuditEventsTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwException_when_creatingAuditEventsWithoutUser() {
+ new AuditEvents(null);
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexCreateRecordEventSuccess() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexCreateRecordSuccessEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Successfully created record in index", payload.get("message"));
+ assertEquals(AuditAction.CREATE, payload.get("action"));
+ assertEquals("IN001", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexCreateRecordEventFail() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexCreateRecordFailEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.FAILURE, payload.get("status"));
+ assertEquals("Failed creating record in index", payload.get("message"));
+ assertEquals(AuditAction.CREATE, payload.get("action"));
+ assertEquals("IN001", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexUpdateRecordEventSuccess() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexUpdateRecordSuccessEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Successfully updated record in index", payload.get("message"));
+ assertEquals(AuditAction.UPDATE, payload.get("action"));
+ assertEquals("IN002", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexUpdateRecordEventFail() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexUpdateRecordFailEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.FAILURE, payload.get("status"));
+ assertEquals("Failed updating record in index", payload.get("message"));
+ assertEquals(AuditAction.UPDATE, payload.get("action"));
+ assertEquals("IN002", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexDeleteRecordEventSuccess() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexDeleteRecordSuccessEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Successfully deleted record in index", payload.get("message"));
+ assertEquals(AuditAction.DELETE, payload.get("action"));
+ assertEquals("IN003", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexDeleteRecordEventFail() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexDeleteRecordFailEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.FAILURE, payload.get("status"));
+ assertEquals("Failed deleting record in index", payload.get("message"));
+ assertEquals(AuditAction.DELETE, payload.get("action"));
+ assertEquals("IN003", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexPurgeRecordEventSuccess() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexPurgeRecordSuccessEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Successfully deleted record in index", payload.get("message"));
+ assertEquals(AuditAction.DELETE, payload.get("action"));
+ assertEquals("IN004", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexPurgeRecordEventFail() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexPurgeRecordFailEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.FAILURE, payload.get("status"));
+ assertEquals("Failed deleting record in index", payload.get("message"));
+ assertEquals(AuditAction.DELETE, payload.get("action"));
+ assertEquals("IN004", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getReindexEvent() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getReindexEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Reindex kind", payload.get("message"));
+ assertEquals(AuditAction.CREATE, payload.get("action"));
+ assertEquals("IN007", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getCopyIndexEvent() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getCopyIndexEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Copy index", payload.get("message"));
+ assertEquals(AuditAction.CREATE, payload.get("action"));
+ assertEquals("IN008", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void getTaskStatusEvent() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getTaskStatusEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Get task status", payload.get("message"));
+ assertEquals(AuditAction.READ, payload.get("action"));
+ assertEquals("IN009", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexCleanUpJobRunEvent() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexCleanUpJobRunEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Index clean-up status job run success", payload.get("message"));
+ assertEquals(AuditAction.JOB_RUN, payload.get("action"));
+ assertEquals("IN010", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexMappingUpdateEventSuccess() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexMappingUpdateEvent(Lists.newArrayList("anything"),true)
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Successfully updated index mapping", payload.get("message"));
+ assertEquals(AuditAction.UPDATE, payload.get("action"));
+ assertEquals("IN0011", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getIndexMappingUpdateEventFail() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getIndexMappingUpdateEvent(Lists.newArrayList("anything"),false)
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.FAILURE, payload.get("status"));
+ assertEquals("Failed updating index mapping", payload.get("message"));
+ assertEquals(AuditAction.UPDATE, payload.get("action"));
+ assertEquals("IN0011", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditLoggerTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditLoggerTest.java
new file mode 100644
index 000000000..09064e70c
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/logging/AuditLoggerTest.java
@@ -0,0 +1,210 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import com.google.common.collect.Lists;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.logging.payload.AuditPayload;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.util.HeadersInfo;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class AuditLoggerTest {
+
+ @Mock
+ private JaxRsDpsLog logger;
+ @Mock
+ private HeadersInfo headers;
+ @InjectMocks
+ private AuditLogger sut;
+
+ @Before
+ public void setup() {
+ when(this.headers.getUser()).thenReturn("testUser");
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexCreateRecordSuccess() {
+ this.sut.indexCreateRecordSuccess(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN001", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexCreateRecordFail() {
+ this.sut.indexCreateRecordFail(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN001", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexUpdateRecordSuccess() {
+ this.sut.indexUpdateRecordSuccess(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN002", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexUpdateRecordFail() {
+ this.sut.indexUpdateRecordFail(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN002", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexDeleteRecordSuccess() {
+ this.sut.indexDeleteRecordSuccess(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN003", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexDeleteRecordFail() {
+ this.sut.indexDeleteRecordFail(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN003", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_getReindex() {
+ this.sut.getReindex(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN007", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_copyIndex() {
+ this.sut.copyIndex(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN008", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_getTaskStatus() {
+ this.sut.getTaskStatus(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN009", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_getIndexCleanUpJobRun() {
+ this.sut.getIndexCleanUpJobRun(Lists.newArrayList("anything"));
+
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+
+ verify(this.logger).audit(payloadCaptor.capture());
+
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN010", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexMappingUpdateFail() {
+ this.sut.indexMappingUpdateFail(Lists.newArrayList("anything"));
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+ verify(this.logger).audit(payloadCaptor.capture());
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_indexMappingUpdateSuccess() {
+ this.sut.indexMappingUpdateSuccess(Lists.newArrayList("anything"));
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+ verify(this.logger).audit(payloadCaptor.capture());
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/logging/JaxRsDpsLogTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/logging/JaxRsDpsLogTest.java
new file mode 100644
index 000000000..04eeb955c
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/logging/JaxRsDpsLogTest.java
@@ -0,0 +1,114 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.logging;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.logging.DpsLog;
+import org.opendes.client.logging.payload.AuditPayload;
+import org.opendes.client.logging.payload.Request;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({DpsLog.class})
+public class JaxRsDpsLogTest {
+
+ @Mock
+ private DpsLog log;
+
+ @InjectMocks
+ private JaxRsDpsLog sut;
+
+ @InjectMocks
+ private ServiceLogId logId;
+
+ @Test
+ public void should_includeAllHeadersExceptAuth_when_writingALog() {
+ this.sut.info("msg");
+
+ ArgumentCaptor<Map> argument = ArgumentCaptor.forClass(Map.class);
+ verify(this.log).info(any(), any(), argument.capture());
+ assertEquals("cor123", argument.getValue().get(DpsHeaders.CORRELATION_ID));
+ assertFalse(argument.getValue().containsKey("authorization"));
+
+ when(logId.getAppLog()).thenReturn("search.app");
+ when(logId.getAuditLog()).thenReturn("search.audit");
+ when(logId.getRequestLog()).thenReturn("search.request");
+ }
+
+ @Test
+ public void should_writeToAuditLogWithGivenPayload_on_auditRequests() {
+ AuditPayload pl = new AuditPayload();
+ this.sut.audit(pl);
+ verify(this.log).audit(eq("search.audit"), eq(pl), any());
+ }
+
+ @Test
+ public void should_writeToRequestLogWithGivenHttpObj_on_requestLog() {
+ Request http = Request.builder().build();
+ this.sut.request(http);
+ verify(this.log).request(eq("search.request"), eq(http), any());
+ }
+
+ @Test
+ public void should_writeToAppLogWithGivenMsg_on_errorLogrequest() {
+ this.sut.error("error");
+ verify(this.log).error(eq("search.app"), eq("error"), any());
+ }
+
+ @Test
+ public void should_writeToAppLogWithGivenMsg_on_warningLogrequest() {
+ this.sut.warning("warning");
+ verify(this.log).warning(eq("search.app"), eq("warning"), any());
+ }
+
+ @Test
+ public void should_writeToAppLogWithGivenMsgArray_on_warningLogrequest() {
+ List<String> warnings = Arrays.asList("Mismatch", "OutOfRange");
+ String output = "0: Mismatch" + System.lineSeparator() + "1: OutOfRange" + System.lineSeparator();
+ this.sut.warning(warnings);
+ verify(this.log).warning(eq("search.app"), eq(output), any());
+ }
+
+ @Test
+ public void should_notWriteToAppLogWithGivenNullMsgArray_on_warningLogrequest() {
+ List<String> warnings = null;
+ this.sut.warning(warnings);
+ verify(this.log, never()).warning(eq("search.app"), eq(null), any());
+ }
+
+ @Test
+ public void should_notWriteToAppLogWithGivenEmptyMsgArray_on_warningLogrequest() {
+ List<String> warnings = new ArrayList<>();
+ this.sut.warning(warnings);
+ verify(this.log, never()).warning(eq("search.app"), eq(""), any());
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/middleware/IndexerFilterTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/middleware/IndexerFilterTest.java
new file mode 100644
index 000000000..1e3b6819e
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/middleware/IndexerFilterTest.java
@@ -0,0 +1,128 @@
+//// Copyright 2017-2019, Schlumberger
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opendes.indexer.middleware;
+//
+//import org.junit.Assert;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.mockito.InjectMocks;
+//import org.mockito.Mock;
+//import org.opendes.client.api.DpsHeaders;
+//import org.opendes.core.auth.AuthorizationService;
+//import org.opendes.core.util.Config;
+//import org.opendes.indexer.util.IRequestInfo;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import org.springframework.test.context.junit4.SpringRunner;
+//import org.springframework.web.bind.annotation.ExceptionHandler;
+//
+//import javax.annotation.security.RolesAllowed;
+//import javax.servlet.FilterChain;
+//import javax.servlet.ServletException;
+//import javax.servlet.ServletRequest;
+//import javax.servlet.ServletResponse;
+//import javax.servlet.http.HttpServletRequest;
+//import javax.ws.rs.container.ContainerRequestContext;
+//import javax.ws.rs.container.ResourceInfo;
+//import javax.ws.rs.core.UriInfo;
+//import java.io.IOException;
+//import java.util.HashMap;
+//import java.util.Map;
+//
+//import static org.mockito.Mockito.mock;
+//import static org.mockito.MockitoAnnotations.initMocks;
+//import static org.powermock.api.mockito.PowerMockito.mockStatic;
+//import static org.powermock.api.mockito.PowerMockito.when;
+//
+//@RunWith(SpringRunner.class)
+//@PrepareForTest({ContainerRequestContext.class, Config.class})
+//public class IndexerFilterTest {
+//
+// private static final String ROLE1 = "role1";
+// private static final String ROLE2 = "role2";
+//
+// @Mock
+// private ServletRequest request;
+// @Mock
+// private AuthorizationService authorizationService;
+// @Mock(name = "resourceInfo")
+// private ResourceInfo resourceInfo;
+//
+// @Mock
+// private IRequestInfo requestInfo;
+// @InjectMocks
+// private IndexerFilter filter;
+//
+// private Map<String, String> headers;
+//
+// @Before
+// public void setup() {
+// initMocks(this);
+//
+// mockStatic(Config.class);
+//
+// headers = new HashMap<>();
+// headers.put(DpsHeaders.ACCOUNT_ID, "tenant1");
+// headers.put(DpsHeaders.AUTHORIZATION, "Bearer geer.fereferv.cefe=");
+// headers.put(DpsHeaders.CONTENT_TYPE, "application/json");
+// headers.put("X-AppEngine-QueueName", "indexer-task-queue");
+// DpsHeaders dpsHeaders = DpsHeaders.createFromMap(headers);
+//
+// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
+// }
+//
+// @Ignore
+// @Test
+// @ExceptionHandler({IOException.class, ServletException.class})
+// public void shouldNot_addAnyHeaders_filterSwaggerPath() throws IOException, ServletException {
+// HttpServletRequest httpRequest = (HttpServletRequest) this.request;
+// ServletResponse response = mock(ServletResponse.class);
+// FilterChain filterChain = mock(FilterChain.class);
+//
+// when(httpRequest.getRequestURI().toLowerCase()).thenReturn("/swagger.json");
+//
+// when(httpRequest.getMethod()).thenReturn("GET");
+//
+// this.filter.doFilter(this.request, response, filterChain);
+// Assert.assertFalse(headers.containsKey(DpsHeaders.CORRELATION_ID));
+// Assert.assertFalse(headers.containsKey(AppEngineHeaders.CLOUD_TRACE_CONTEXT));
+// }
+//
+// @Ignore
+// @Test
+// @ExceptionHandler({IOException.class, ServletException.class})
+// public void shouldNot_addAnyHeaders_filterIndexerPath() throws IOException, ServletException {
+// HttpServletRequest httpRequest = (HttpServletRequest) this.request;
+// ServletResponse response = mock(ServletResponse.class);
+// FilterChain filterChain = mock(FilterChain.class);
+//
+// UriInfo urlInfo = mock(UriInfo.class);
+// when(httpRequest.getRequestURI().toLowerCase()).thenReturn("task-handlers");
+//
+// when(httpRequest.getMethod()).thenReturn("POST");
+//
+// when(this.requestInfo.isTaskQueueRequest()).thenReturn(true);
+//
+// this.filter.doFilter(this.request, response, filterChain);
+// Assert.assertFalse(headers.containsKey(DpsHeaders.CORRELATION_ID));
+// Assert.assertFalse(headers.containsKey(AppEngineHeaders.CLOUD_TRACE_CONTEXT));
+// }
+//
+// @RolesAllowed({ROLE1, ROLE2})
+// public void rolesAllowedTestMethod() {
+// // do nothing
+// }
+//}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandlerTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandlerTest.java
new file mode 100644
index 000000000..49962a701
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/middleware/RedirectHttpRequestsHandlerTest.java
@@ -0,0 +1,104 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.middleware;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.util.IRequestInfo;
+import org.springframework.test.context.junit4.SpringRunner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.core.SecurityContext;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class RedirectHttpRequestsHandlerTest {
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private ContainerRequestContext context;
+ @Mock
+ private SecurityContext securityContext;
+ @Mock
+ private javax.inject.Provider<IRequestInfo> requestInfoProvider;
+ @InjectMocks
+ private RedirectHttpRequestsHandler sut;
+
+ @Before
+ public void setup() {
+ when(requestInfoProvider.get()).thenReturn(requestInfo);
+ }
+
+ @Test
+ public void should_throwAppException302WithHttpsLocation_when_isNotACron_And_IsNotUsingHttps() {
+ when(requestInfo.isCronRequest()).thenReturn(false);
+ when(context.getSecurityContext()).thenReturn(securityContext);
+ when(securityContext.isSecure()).thenReturn(false);
+
+ try {
+ sut.filter(context);
+ fail("should throw");
+ } catch (AppException e) {
+ assertEquals(302, e.getError().getCode());
+ }
+ }
+
+ @Test
+ public void should_throwAppException302WithHttpsLocation_when_isNotATaskQueue_And_IsNotUsingHttps() {
+ when(requestInfo.isTaskQueueRequest()).thenReturn(false);
+ when(context.getSecurityContext()).thenReturn(securityContext);
+ when(securityContext.isSecure()).thenReturn(false);
+
+ try {
+ sut.filter(context);
+ fail("should throw");
+ } catch (AppException e) {
+ assertEquals(302, e.getError().getCode());
+ }
+ }
+
+ @Test
+ public void should_notThrowAppException302WithHttpsLocation_when_isACron() {
+ when(requestInfo.isCronRequest()).thenReturn(true);
+ when(context.getSecurityContext()).thenReturn(securityContext);
+ when(securityContext.isSecure()).thenReturn(false);
+
+ sut.filter(context);
+ }
+
+ @Test
+ public void should_notThrowAppException302WithHttpsLocation_when_isAHttpsRequest() {
+ when(requestInfo.isCronRequest()).thenReturn(false);
+ when(context.getSecurityContext()).thenReturn(securityContext);
+ when(securityContext.isSecure()).thenReturn(true);
+
+ sut.filter(context);
+ }
+
+ @Test
+ public void should_notThrowAppException302WithHttpsLocation_when_isATaskQueue() {
+ when(requestInfo.isTaskQueueRequest()).thenReturn(true);
+ when(context.getSecurityContext()).thenReturn(securityContext);
+ when(securityContext.isSecure()).thenReturn(false);
+
+ sut.filter(context);
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/model/ElasticTypeTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/model/ElasticTypeTest.java
new file mode 100644
index 000000000..9fc257b5e
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/model/ElasticTypeTest.java
@@ -0,0 +1,51 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class ElasticTypeTest {
+
+ @Test
+ public void should_returnUndefined_givenNull_forValueTest() {
+ Assert.assertEquals(ElasticType.UNDEFINED, ElasticType.forValue(null));
+ }
+
+ @Test
+ public void should_returnUndefined_givenUnknownType_forValueTest() {
+ Assert.assertEquals(ElasticType.UNDEFINED, ElasticType.forValue("test"));
+ }
+
+ @Test
+ public void should_returnCorrespondingType_givenValidType_forValueTest() {
+ Assert.assertEquals(ElasticType.KEYWORD, ElasticType.forValue("keyword"));
+ Assert.assertEquals(ElasticType.TEXT, ElasticType.forValue("Text"));
+ Assert.assertEquals(ElasticType.DATE, ElasticType.forValue("Date"));
+ Assert.assertEquals(ElasticType.NESTED, ElasticType.forValue("nestED"));
+ Assert.assertEquals(ElasticType.OBJECT, ElasticType.forValue("OBJECT"));
+ Assert.assertEquals(ElasticType.GEO_POINT, ElasticType.forValue("geo_point"));
+ Assert.assertEquals(ElasticType.GEO_SHAPE, ElasticType.forValue("geo_shape"));
+ Assert.assertEquals(ElasticType.INTEGER, ElasticType.forValue("INteger"));
+ Assert.assertEquals(ElasticType.LONG, ElasticType.forValue("long"));
+ Assert.assertEquals(ElasticType.FLOAT, ElasticType.forValue("FLOAT"));
+ Assert.assertEquals(ElasticType.DOUBLE, ElasticType.forValue("DOUBLE"));
+ Assert.assertEquals(ElasticType.BOOLEAN, ElasticType.forValue("Boolean"));
+ Assert.assertEquals(ElasticType.UNDEFINED, ElasticType.forValue("undefined"));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/model/IndexingStatusTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/model/IndexingStatusTest.java
new file mode 100644
index 000000000..5ad06c19a
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/model/IndexingStatusTest.java
@@ -0,0 +1,44 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class IndexingStatusTest {
+
+ @Test
+ public void should_returnTrue_givenCorrectParameterOrder_isWorseThanTest() {
+ Assert.assertTrue(IndexingStatus.FAIL.isWorseThan(IndexingStatus.SKIP));
+ Assert.assertTrue(IndexingStatus.FAIL.isWorseThan(IndexingStatus.SUCCESS));
+ Assert.assertTrue(IndexingStatus.FAIL.isWorseThan(IndexingStatus.PROCESSING));
+ Assert.assertTrue(IndexingStatus.SKIP.isWorseThan(IndexingStatus.SUCCESS));
+ Assert.assertTrue(IndexingStatus.SKIP.isWorseThan(IndexingStatus.PROCESSING));
+ Assert.assertTrue(IndexingStatus.SUCCESS.isWorseThan(IndexingStatus.PROCESSING));
+ }
+
+ @Test
+ public void should_returnFalse_givenReversedParameterOrder_isWorseThanTest() {
+ Assert.assertFalse(IndexingStatus.PROCESSING.isWorseThan(IndexingStatus.SUCCESS));
+ Assert.assertFalse(IndexingStatus.PROCESSING.isWorseThan(IndexingStatus.SKIP));
+ Assert.assertFalse(IndexingStatus.PROCESSING.isWorseThan(IndexingStatus.FAIL));
+ Assert.assertFalse(IndexingStatus.SUCCESS.isWorseThan(IndexingStatus.SKIP));
+ Assert.assertFalse(IndexingStatus.SUCCESS.isWorseThan(IndexingStatus.FAIL));
+ Assert.assertFalse(IndexingStatus.SKIP.isWorseThan(IndexingStatus.FAIL));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/model/RecordChagedMessagesTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/model/RecordChagedMessagesTest.java
new file mode 100644
index 000000000..a33c442f1
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/model/RecordChagedMessagesTest.java
@@ -0,0 +1,91 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.model.RecordChangedMessages;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+@RunWith(SpringRunner.class)
+public class RecordChagedMessagesTest {
+
+ private RecordChangedMessages recordChangedMessages;
+ private final String ACCOUNT_ID = "test-tenant";
+ private final String CORRELATION_ID = "xxxxxx";
+
+ @Before
+ public void setup() {
+ Map<String, String> headers = new HashMap<>();
+ headers.put(DpsHeaders.ACCOUNT_ID, ACCOUNT_ID);
+ headers.put(DpsHeaders.CORRELATION_ID, CORRELATION_ID);
+ recordChangedMessages = RecordChangedMessages.builder()
+ .attributes(headers).build();
+ }
+
+ @Test
+ public void should_returnCorrectAccountID_getAccountIdTest() {
+ assertEquals(ACCOUNT_ID, recordChangedMessages.getDataPartitionId());
+ }
+
+ @Test
+ public void should_returnFalse_ifThereIsAccountID_missingAccountIdTest() {
+ assertFalse(recordChangedMessages.missingAccountId());
+ }
+
+ @Test
+ public void should_returnTrue_ifThereIsNoAccountID_missingAccountIdTest() {
+ recordChangedMessages.setAttributes(null);
+ assertTrue(recordChangedMessages.missingAccountId());
+ }
+
+ @Test
+ public void should_returnCorrectCorrelationId_getCorrelationIdTest() {
+ assertEquals(CORRELATION_ID, recordChangedMessages.getCorrelationId());
+ }
+
+ @Test
+ public void should_returnTrue_ifThereIsCorrelationId_hasCorrelationIdTest() {
+ Assert.assertTrue(recordChangedMessages.hasCorrelationId());
+ }
+
+ @Test
+ public void should_returnFalse_ifThereIsNoCorrelationId_hasCorrelationIdTest() {
+ RecordChangedMessages recordMessages = new RecordChangedMessages();
+ assertFalse(recordMessages.hasCorrelationId());
+ }
+
+ @Test
+ public void should_returnFalse_ifEmptyAccountIDTest() {
+ Map<String, String> headers = new HashMap<>();
+ headers.put(DpsHeaders.ACCOUNT_ID, "");
+ headers.put(DpsHeaders.CORRELATION_ID, "");
+ RecordChangedMessages invalidMessages = RecordChangedMessages.builder()
+ .attributes(headers).build();
+
+ assertTrue(invalidMessages.missingAccountId());
+ assertFalse(invalidMessages.hasCorrelationId());
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/model/StorageTypeTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/model/StorageTypeTest.java
new file mode 100644
index 000000000..3cc673d1b
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/model/StorageTypeTest.java
@@ -0,0 +1,40 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.model;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class StorageTypeTest {
+
+ @Test
+ public void should_returnCorrectValue_getValueTest() {
+ Assert.assertEquals("link", StorageType.LINK.getValue());
+ Assert.assertEquals("[]link", StorageType.LINK_ARRAY.getValue());
+ Assert.assertEquals("boolean", StorageType.BOOLEAN.getValue());
+ Assert.assertEquals("string", StorageType.STRING.getValue());
+ Assert.assertEquals("int", StorageType.INT.getValue());
+ Assert.assertEquals("float", StorageType.FLOAT.getValue());
+ Assert.assertEquals("double", StorageType.DOUBLE.getValue());
+ Assert.assertEquals("[]double", StorageType.DOUBLE_ARRAY.getValue());
+ Assert.assertEquals("long", StorageType.LONG.getValue());
+ Assert.assertEquals("datetime", StorageType.DATETIME.getValue());
+ Assert.assertEquals("core:dl:geopoint:1.0.0", StorageType.GEO_POINT.getValue());
+ Assert.assertEquals("core:dl:geoshape:1.0.0", StorageType.GEO_SHAPE.getValue());
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/AttributeParsingServiceImplTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/AttributeParsingServiceImplTest.java
new file mode 100644
index 000000000..08276951d
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/AttributeParsingServiceImplTest.java
@@ -0,0 +1,219 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.internal.LinkedTreeMap;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.indexer.util.parser.DateTimeParser;
+import org.opendes.indexer.util.parser.NumberParser;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class AttributeParsingServiceImplTest {
+
+ @Mock
+ private GeometryConversionService geometryConversionService;
+ @Mock
+ private NumberParser numberParser;
+ @Mock
+ private DateTimeParser dateTimeParser;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private AttributeParsingServiceImpl sut;
+
+ @Test
+ public void should_parseValidInteger() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ when(this.numberParser.parseInteger(any(), any())).thenThrow(new IllegalArgumentException("number parsing error, integer out of range: attribute: lat | value: 101959.E1019594E"));
+ this.sut.tryParseInteger("common:welldb:wellbore-OGY4ZWQ5", "lat", "101959.E1019594E", dataMap);
+ assertEquals(dataMap.size(), 0);
+ assertFalse(dataMap.containsKey("lat"));
+ }
+
+ @Test
+ public void should_parseValidLong() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ when(this.numberParser.parseLong(any(), any())).thenReturn(0L);
+ this.sut.tryParseLong("common:welldb:wellbore-OGY4ZWQ5", "reference", "", dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertEquals(dataMap.get("reference"), 0L);
+ }
+
+ @Test
+ public void should_parseValidFloat() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ when(this.numberParser.parseFloat(any(), any())).thenReturn(0f);
+ this.sut.tryParseFloat("common:welldb:wellbore-MjVhND", "lon", null, dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertEquals(dataMap.get("lon"), 0.0f);
+ }
+
+ @Test
+ public void should_parseValidDouble() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ when(this.numberParser.parseDouble(any(), any())).thenReturn(20.0);
+ this.sut.tryParseDouble("common:welldb:wellbore-zMWQtMm", "location", 20.0, dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertEquals(dataMap.get("location"), 20.0);
+ }
+
+ @Test
+ public void should_parseBoolean() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseBoolean("common:welldb:wellbore-OGY4ZWQ5", "dry", "", dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertEquals(dataMap.get("dry"), false);
+
+ this.sut.tryParseBoolean("common:welldb:wellbore-OGY4ZWQ5", "active", null, dataMap);
+ assertEquals(dataMap.size(), 2);
+ assertEquals(dataMap.get("active"), false);
+
+ this.sut.tryParseBoolean("common:welldb:wellbore-OGY4ZWQ5", "notation", "E.2131", dataMap);
+ assertEquals(dataMap.size(), 3);
+ assertEquals(dataMap.get("notation"), false);
+
+ this.sut.tryParseBoolean("common:welldb:wellbore-OGY4ZWQ5", "aw", false, dataMap);
+ assertEquals(dataMap.size(), 4);
+ assertEquals(dataMap.get("aw"), false);
+
+ this.sut.tryParseBoolean("common:welldb:wellbore-OGY4ZWQ5", "side", "true", dataMap);
+ assertEquals(dataMap.size(), 5);
+ assertEquals(dataMap.get("side"), true);
+ }
+
+ @Test
+ public void should_parseDate_tryParseDate() {
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseDate("common:welldb:wellbore-OGY4ZWQ5", "createTime", "", dataMap);
+ assertEquals(dataMap.size(), 0);
+ assertFalse(dataMap.containsKey("createTime"));
+
+ this.sut.tryParseDate("common:welldb:wellbore-OGY4ZWQ5", "activatedOn", null, dataMap);
+ assertEquals(dataMap.size(), 0);
+ assertFalse(dataMap.containsKey("activatedOn"));
+
+ this.sut.tryParseDate("common:welldb:wellbore-OGY4ZWQ5", "activatedOn", "E.2131", dataMap);
+ assertEquals(dataMap.size(), 0);
+ assertFalse(dataMap.containsKey("activatedOn"));
+
+ when(this.dateTimeParser.convertDateObjectToUtc("2018-11-06T19:37:11.128Z")).thenReturn("2018-11-06T19:37:11+0000");
+ this.sut.tryParseDate("common:welldb:wellbore-OGY4ZWQ5", "disabledOn", "2018-11-06T19:37:11.128Z", dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertTrue(dataMap.containsKey("disabledOn"));
+ assertEquals(dataMap.get("disabledOn"), "2018-11-06T19:37:11+0000");
+ }
+
+ @Test
+ public void should_notReturnLatLong_given_oneOfTheNullAttribute_tryGetGeopointTest() {
+ LinkedTreeMap<String, Object> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", null);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("location", positionTreeMap);
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseGeopoint("common:welldb:wellbore-NjdhZTZ", "location", storageData, dataMap);
+
+ assertTrue(dataMap.isEmpty());
+ }
+
+ @Test
+ public void should_notReturnLatLong_given_oneOfTheEmptyAttribute_tryGetGeopointTest() {
+ LinkedTreeMap<String, Object> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", 23.46);
+ positionTreeMap.put("latitude", "");
+
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("location", positionTreeMap);
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseGeopoint("common:welldb:wellbore-NjdhZTZ", "location", storageData, dataMap);
+
+ assertTrue(dataMap.isEmpty());
+ }
+
+ @Test
+ public void should_notReturnLatLong_given_invalidTreeMap_tryGetGeopointTest() {
+ LinkedTreeMap<String, Object> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", "hello");
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("location", positionTreeMap);
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseGeopoint("", "location", storageData, dataMap);
+
+ assertTrue(dataMap.isEmpty());
+ }
+
+ @Test
+ public void should_notReturnLatLong_given_outOfRange_tryGetGeopointTest() {
+ LinkedTreeMap<String, Object> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", -189);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("location", positionTreeMap);
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseGeopoint("", "location", storageData, dataMap);
+
+ assertTrue(dataMap.isEmpty());
+ }
+
+ @Test
+ public void should_returnLatLong_given_validTreeMap_tryGetGeopointTest() {
+ Map<String, Double> positionMap = new HashMap<>();
+ positionMap.put("longitude", 10.45);
+ positionMap.put("latitude", 90.0);
+
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("location", positionMap);
+
+ when(this.geometryConversionService.tryGetGeopoint(positionMap)).thenReturn(positionMap);
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ this.sut.tryParseGeopoint("", "location", storageData, dataMap);
+
+ assertFalse(dataMap.isEmpty());
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/CronServiceImplTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/CronServiceImplTest.java
new file mode 100644
index 000000000..f80e99dfc
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/CronServiceImplTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.common.collect.Lists;
+import com.sun.corba.se.impl.interceptors.RequestInfoImpl;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.model.IndexInfo;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.indexer.util.IRequestInfo;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+
+import static org.mockito.Mockito.*;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Config.class})
+public class CronServiceImplTest {
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private CronServiceImpl sut;
+
+ @InjectMocks
+ private DpsHeaders dpsHeaders;
+
+ @Before
+ public void setup() {
+ mockStatic(Config.class);
+
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+
+ when(Config.getIndexCleanupThresholdDays()).thenReturn(3);
+ when(Config.getEmptyIndexCleanupThresholdDays()).thenReturn(7);
+ }
+
+ @Test
+ public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
+ final String indexPattern = "tenant1-index-*";
+
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupIndices(indexPattern);
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception);
+
+ this.sut.cleanupIndices("tenant1-test-*");
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+
+ @Test
+ public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test
+ public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_backend_throws_exception() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/GeometryConversionServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/GeometryConversionServiceTest.java
new file mode 100644
index 000000000..34fd167f0
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/GeometryConversionServiceTest.java
@@ -0,0 +1,124 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.internal.LinkedTreeMap;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.opendes.core.util.Constants;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+@RunWith(SpringRunner.class)
+public class GeometryConversionServiceTest {
+
+ @InjectMocks
+ private GeometryConversionService sut;
+
+ @Test
+ public void should_returnValidResults_given_validTreeMap_getGeoShapeTest() {
+ LinkedTreeMap<String, Object> map = new LinkedTreeMap<>();
+ List<LinkedTreeMap<String, Object>> features = new ArrayList<>();
+ LinkedTreeMap<String, Object> object = new LinkedTreeMap<>();
+ LinkedTreeMap<String, Object> geometry = new LinkedTreeMap<>();
+ LinkedTreeMap<String, Object> properties = new LinkedTreeMap<>();
+ properties.put("dummyKye", "dummyValue");
+ geometry.put("type", "type");
+ geometry.put("radius", "radius");
+ geometry.put("coordinates", "coordinates");
+ object.put("geometry", geometry);
+ object.put(Constants.PROPERTIES, properties);
+ features.add(object);
+ map.put("features", features);
+
+ List<Map<String, Object>> geometries = this.sut.getGeoShape(map);
+
+ assertEquals(1, geometries.size());
+ assertEquals(properties, geometries.get(0).get(Constants.PROPERTIES));
+ }
+
+ @Test
+ public void should_returnNull_given_nullOrEmptyTreeMap_tryGetGeopointTest() {
+ assertNull(this.sut.tryGetGeopoint(null));
+ assertNull(this.sut.tryGetGeopoint(new LinkedTreeMap<>()));
+ }
+
+ @Test
+ public void should_returnNull_given_invalidTreeMap_tryGetGeopointTest() {
+ LinkedTreeMap<String, Double> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", null);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Double> result = this.sut.tryGetGeopoint(positionTreeMap);
+ assertNull(result);
+ }
+
+ @Test
+ public void should_returnLatLong_given_validTreeMap_tryGetGeopointTest() {
+ LinkedTreeMap<String, Double> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", 10.0);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Double> result = this.sut.tryGetGeopoint(positionTreeMap);
+
+ assertEquals(new Double(10.0), result.get("lon"));
+ assertEquals(new Double(20.0), result.get("lat"));
+ }
+
+ @Test
+ public void should_throwNullPointerException_given_nullTreeMap_getGeopointGeoJsonTest() {
+ Map<String, Object> geometry = this.sut.getGeopointGeoJson(null);
+
+ assertNull(geometry);
+ }
+
+ @Test
+ public void should_returnValidResults_given_validTreeMap_getGeopointGeoJsonTest() {
+ LinkedTreeMap<String, Double> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", 10.0);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Object> result = this.sut.getGeopointGeoJson(positionTreeMap);
+
+ assertEquals(2, result.size());
+ assertEquals("geometrycollection", result.get(Constants.TYPE));
+ }
+
+ @Test
+ public void should_throwNullPointerException_given_nullTreeMap_getGeopointGeometryTest() {
+ Map<String, Object> geometry = this.sut.getGeopointGeometry(null);
+
+ assertNull(geometry);
+ }
+
+ @Test
+ public void should_returnValidResults_given_validTreeMap_getGeopointGeometryTest() {
+ LinkedTreeMap<String, Double> positionTreeMap = new LinkedTreeMap<>();
+ positionTreeMap.put("longitude", 10.0);
+ positionTreeMap.put("latitude", 20.0);
+
+ Map<String, Object> result = this.sut.getGeopointGeometry(positionTreeMap);
+
+ assertEquals(2, result.size());
+ assertEquals("point", result.get(Constants.TYPE));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexCopyServiceImplTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexCopyServiceImplTest.java
new file mode 100644
index 000000000..1af6fcfbe
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexCopyServiceImplTest.java
@@ -0,0 +1,184 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.model.ClusterSettings;
+import org.opendes.core.service.ElasticSettingService;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.core.util.HeadersInfo;
+import org.opendes.indexer.logging.AuditLogger;
+import org.opendes.indexer.util.IRequestInfo;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class})
+public class IndexCopyServiceImplTest {
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private HttpEntity httpEntity;
+ @Mock
+ private HttpEntity httpEntityRequest;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private HeadersInfo headersInfo;
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private Response response;
+ @Mock
+ private ElasticSettingService elasticSettingService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private Map<String, String> httpHeaders;
+ @InjectMocks
+ private IndexCopyServiceImpl sut;
+
+ private ClusterSettings commonCluster;
+
+ private Map<String, Object> correctMap;
+
+ @Before
+ public void setup() {
+// mockStatic(EntityUtils.class);
+ commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(headersInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(response.getEntity()).thenReturn(httpEntity);
+
+ Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
+ String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
+ correctMap = new Gson().fromJson(afterFormat, mapType);
+ }
+
+ @Test(expected = IOException.class)
+ public void should_throwIOException_when_indexMappingNotFound() throws Exception {
+ IOException exception = new IOException("Fail to get mapping for the given index from common cluster.");
+
+ when(this.mappingService.getIndexMapping(any(), any())).thenThrow(exception);
+
+ this.sut.copyIndex("common:metadata:entity:1.0.0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
+ try {
+ this.sut.copyIndex(null);
+ } catch (IOException e) {
+ fail("Should not throw IOException but illegalArgumentException.");
+ }
+ }
+
+ @Test
+ public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
+ String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ try {
+ when(this.mappingService.getIndexMapping(any(), any())).thenReturn(mappingJson);
+ Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
+ Assert.assertEquals(resultMap, correctMap);
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void should_returnClusterInfo_getCommonClusterInformationtest() {
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
+ String[] resultCommonCluster = this.sut.getCommonClusterInformation();
+ Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
+ Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
+ Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
+ } catch (IOException ignored) {
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
+ try {
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(false);
+ this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
+ } catch (IOException ignored) {
+ }
+ }
+
+ @Ignore
+ public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
+ //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(httpEntityRequest);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(false);
+ when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(restClient.performRequest(request)).thenReturn(response);
+ when(response.getEntity()).thenReturn(httpEntity);
+ Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
+ } catch (IOException ignored) {
+ }
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerMappingServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerMappingServiceTest.java
new file mode 100644
index 000000000..12b31738c
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerMappingServiceTest.java
@@ -0,0 +1,324 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.spy;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.IndicesClient;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.model.RecordMetaAttribute;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.indexer.model.IndexSchema;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+
+import org.springframework.test.context.junit4.SpringRunner;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ Map<String, String> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerSchemaServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerSchemaServiceTest.java
new file mode 100644
index 000000000..5f64c8fc4
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerSchemaServiceTest.java
@@ -0,0 +1,325 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.httpclient.RequestStatus;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.service.IndicesService;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.ElasticClientHandler;
+import org.opendes.core.util.ElasticIndexNameResolver;
+import org.opendes.indexer.cache.SchemaCache;
+import org.opendes.indexer.model.IndexSchema;
+import org.opendes.indexer.model.OperationType;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private SchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerServiceTest.java
new file mode 100644
index 000000000..4e67a2a9b
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/IndexerServiceTest.java
@@ -0,0 +1,337 @@
+//// Copyright 2017-2019, Schlumberger
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opendes.indexer.service;
+//
+//import com.google.gson.Gson;
+//import com.google.gson.reflect.TypeToken;
+//import org.elasticsearch.action.bulk.BulkItemResponse;
+//import org.elasticsearch.action.bulk.BulkResponse;
+//import org.elasticsearch.client.RequestOptions;
+//import org.elasticsearch.client.RestHighLevelClient;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.mockito.InjectMocks;
+//import org.mockito.Mock;
+//import org.mockito.Spy;
+//import org.opendes.client.api.DpsHeaders;
+//import org.opendes.core.logging.JaxRsDpsLog;
+//import org.opendes.core.model.DeploymentEnvironment;
+//import org.opendes.core.model.RecordChangedMessages;
+//import org.opendes.core.service.IndicesService;
+//import org.opendes.core.util.Config;
+//import org.opendes.core.util.ElasticClientHandler;
+//import org.opendes.core.util.ElasticIndexNameResolver;
+//import org.opendes.core.util.HeadersUtil;
+//import org.opendes.indexer.logging.AuditLogger;
+//import org.opendes.indexer.model.*;
+//import org.opendes.indexer.publish.IPublisher;
+//import org.opendes.indexer.util.IRequestInfo;
+//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+//import org.opendes.indexer.util.JobStatus;
+//import org.opendes.indexer.util.RecordInfo;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import org.springframework.beans.factory.annotation.Autowired;
+//import org.springframework.context.annotation.Lazy;
+//import org.springframework.test.context.junit4.SpringRunner;
+//
+//import java.io.IOException;
+//import java.lang.reflect.Type;
+//import java.util.*;
+//
+//import static java.util.Collections.singletonList;
+//import static org.junit.Assert.*;
+//import static org.mockito.Matchers.any;
+//import static org.mockito.Mockito.verify;
+//import static org.mockito.Mockito.when;
+//import static org.powermock.api.mockito.PowerMockito.mock;
+//import static org.powermock.api.mockito.PowerMockito.mockStatic;
+//
+//@Ignore
+//@RunWith(SpringRunner.class)
+//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
+//public class IndexerServiceTest {
+//
+// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
+// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
+// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
+// private final String recordId1 = "tenant1:doc:test1";
+// private final String recordId2 = "tenant1:doc:test2";
+// private final String failureMassage = "test failure";
+//
+// @Mock
+// private IndexSchemaService indexSchemaService;
+// @Mock
+// private IndicesService indicesService;
+// @Mock
+// private IndexerMappingService indexerMappingService;
+// @Mock
+// private StorageService storageService;
+// @Mock
+// private IPublisher publisherImpl;
+// @Mock
+// private RestHighLevelClient restHighLevelClient;
+// @Mock
+// private ElasticClientHandler elasticClientHandler;
+// @Mock
+// private BulkResponse bulkResponse;
+// @Mock
+// private IRequestInfo requestInfo;
+// @Mock
+// private ElasticIndexNameResolver elasticIndexNameResolver;
+// @Mock
+// private AttributeParsingServiceImpl attributeParsingServiceImpl;
+// @Mock
+// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+// @Mock
+// private JaxRsDpsLog log;
+// @Mock
+// private AuditLogger auditLogger;
+// @InjectMocks
+// private IndexerServiceImpl sut;
+// @InjectMocks @Spy
+// private JobStatus jobStatus = new JobStatus();
+//
+// @Autowired
+// @Lazy
+// private DpsHeaders dpsHeaders;
+// private RecordChangedMessages recordChangedMessages;
+// private List<RecordInfo> recordInfos;
+//
+// @Before
+// public void setup() throws IOException {
+//
+// mockStatic(StorageAcl.class);
+// mockStatic(Config.class);
+//
+// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
+// when(Config.getElasticServerAddress()).thenReturn("testsite");
+//
+// dpsHeaders = new DpsHeaders();
+// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
+// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
+// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+//
+// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+//
+// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
+//
+// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
+// when(bulkResponse.getItems()).thenReturn(responses);
+// Map<String, String> attr = new HashMap<>();
+// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
+// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+// when(StorageAcl.flattenAcl(any())).thenReturn(null);
+// }
+//
+// @Test
+// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
+//
+// assertNull(jobStatus);
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatuses = new LinkedList<>();
+// List<String> status=new ArrayList<>();
+// status.add("crs bla bla");
+// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
+// conversionStatuses.add(conversionStatus);
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
+// try {
+// List<Records.Entity> validRecords = new ArrayList<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+//
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind1, schema);
+// indexSchemaServiceMock(kind2, null);
+// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+//
+// storageData.put("schema1", "test-value");
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+//
+// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
+// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// private BulkItemResponse prepareResponseFail() {
+// BulkItemResponse responseFail = mock(BulkItemResponse.class);
+// when(responseFail.isFailed()).thenReturn(true);
+// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
+// when(responseFail.getId()).thenReturn(recordId1);
+// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
+// return responseFail;
+// }
+//
+// private BulkItemResponse prepareResponseSuccess() {
+// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
+// when(responseSuccess.getId()).thenReturn(recordId2);
+// return responseSuccess;
+// }
+//
+// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
+// if (schema == null) {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// } else {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// }
+// }
+//
+// private Map<String, String> createSchema() {
+// Map<String, String> schema = new HashMap<>();
+// schema.put("schema1", "keyword");
+// schema.put("schema2", "boolean");
+// schema.put("schema3", "date");
+// schema.put("schema6", "object");
+// return schema;
+// }
+//}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/ReindexServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/ReindexServiceTest.java
new file mode 100644
index 000000000..b700b2c64
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/ReindexServiceTest.java
@@ -0,0 +1,144 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.indexer.model.RecordQueryResponse;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.util.IRequestInfo;
+import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+
+ @Mock
+ private Map<String, String> httpHeaders;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/service/StorageServiceTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/service/StorageServiceTest.java
new file mode 100644
index 000000000..1e18790e4
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/service/StorageServiceTest.java
@@ -0,0 +1,210 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opendes.core.httpclient.HttpResponse;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.core.service.UrlFetchService;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.RecordQueryResponse;
+import org.opendes.indexer.model.RecordReindexRequest;
+import org.opendes.indexer.model.Records;
+import org.opendes.indexer.util.IRequestInfo;
+import org.opendes.indexer.util.JobStatus;
+import org.opendes.indexer.util.RecordInfo;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private UrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/util/JobStatusTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/util/JobStatusTest.java
new file mode 100644
index 000000000..39a6ae163
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/util/JobStatusTest.java
@@ -0,0 +1,239 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opendes.core.logging.JaxRsDpsLog;
+import org.opendes.indexer.model.IndexingStatus;
+import org.opendes.indexer.model.OperationType;
+import org.opendes.indexer.model.RecordStatus;
+import org.opendes.indexer.model.Records;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.*;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class JobStatusTest {
+
+ private final String recordChangedMessages = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer12:well:1.0.0\",\"op\":\"create\"}]";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private JobStatus sut;
+
+ @Test
+ public void should_create_emptyStatusList_given_emptyPubSubInfoList_ConstructorTest() {
+ List<RecordInfo> recordInfos = new ArrayList<>();
+ this.sut.initialize(recordInfos);
+ assertEquals(0, this.sut.getStatusesList().size());
+ }
+
+ @Test
+ public void should_create_twoElementsStatusList_given_twoElementsPubSubInfoList_ConstructorTest() {
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = (new Gson()).fromJson(recordChangedMessages, listType);
+ this.sut.initialize(recordInfos);
+ assertEquals(2, this.sut.getStatusesList().size());
+ }
+
+ @Test
+ public void should_get_kind_given_twoElementsPubSubInfoList() {
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = (new Gson()).fromJson(recordChangedMessages, listType);
+ this.sut.initialize(recordInfos);
+
+ String kind = this.sut.getRecordKindById("tenant1:doc:test2");
+ assertNotNull(kind);
+ assertEquals(kind, "tenant1:testindexer12:well:1.0.0");
+ }
+
+
+ @Test
+ public void should_return_emptyStatusList_given_emptyIdCollection_addOrUpdateRetryRecordStatusTest() {
+ this.sut.addOrUpdateRecordStatus(new ArrayList<>(), IndexingStatus.SUCCESS, 1, "");
+
+ assertEquals(0, this.sut.getStatusesList().size());
+ }
+
+
+ @Test
+ public void should_return_emptyStatusList_given_nullIdCollection_addOrUpdateRecordStatusTest() {
+ this.sut.addOrUpdateRecordStatus((Collection<String>) null, IndexingStatus.SUCCESS, 1, "");
+
+ assertEquals(0, this.sut.getStatusesList().size());
+ }
+
+ @Test
+ public void should_return_emptyStatusList_given_emptyIdCollection_addOrUpdateRecordStatusTest() {
+ this.sut.addOrUpdateRecordStatus(new ArrayList<>(), IndexingStatus.SUCCESS, 1, "");
+
+ assertEquals(0, this.sut.getStatusesList().size());
+ }
+
+ @Test
+ public void should_return_fourElementsStatusList_given_fourValidIds_addOrUpdateRecordStatusTest() {
+ List<String> invalidRecords = new ArrayList<>();
+ invalidRecords.add("0001abc@#$");
+ invalidRecords.add("0001abc@#$...");
+ invalidRecords.add("0001abc@#$/01");
+ invalidRecords.add("0001abc@#$.../02");
+ Records records = Records.builder().notFound(invalidRecords).build();
+ Collection<String> ids = records.getNotFound();
+
+ this.sut.addOrUpdateRecordStatus(ids, IndexingStatus.SUCCESS, 1, "");
+ assertEquals(4, this.sut.getStatusesList().size());
+ }
+
+ @Test
+ public void should_not_updateMessage_given_nullOrEmptyMessage_updateRecordStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+ jobStatus.addOrUpdateRecordStatus("success1", IndexingStatus.FAIL, 1, null);
+ assertEquals(4, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals("", jobStatus.getStatusesList().get(1).getLatestTrace());
+ jobStatus.addOrUpdateRecordStatus("success2", IndexingStatus.SKIP, 1, "");
+ assertEquals(5, jobStatus.getIdsByIndexingStatus(IndexingStatus.SKIP).size());
+ assertEquals("", jobStatus.getStatusesList().get(2).getLatestTrace());
+ }
+
+ @Test
+ public void should_updateMessage_given_validMessage_updateRecordStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+ String statusChangeMessage = "unit test status change";
+ jobStatus.addOrUpdateRecordStatus("success1", IndexingStatus.FAIL, 1, statusChangeMessage);
+ assertEquals(statusChangeMessage, jobStatus.getStatusesList().get(1).getLatestTrace());
+ }
+
+ @Test
+ public void should_not_updateStatus_given_higherStatus_updateRecordStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+ String statusChangeMessage = "unit test status change";
+ jobStatus.addOrUpdateRecordStatus("fail1", IndexingStatus.SUCCESS, 1, statusChangeMessage);
+ assertEquals(2, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+ assertEquals(3, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals(statusChangeMessage, jobStatus.getStatusesList().get(3).getLatestTrace());
+ }
+
+ @Test
+ public void should_updateStatus_given_lowerStatus_updateRecordStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+ String statusChangeMessage = "unit test status change";
+ jobStatus.addOrUpdateRecordStatus("success1", IndexingStatus.SKIP, 1, statusChangeMessage);
+ assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+ assertEquals(5, jobStatus.getIdsByIndexingStatus(IndexingStatus.SKIP).size());
+ assertEquals(statusChangeMessage, jobStatus.getStatusesList().get(1).getLatestTrace());
+ }
+
+ @Test
+ public void should_returnValidResponse_getIdsByIndexingStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+ assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.PROCESSING).size());
+ assertEquals(2, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+ assertEquals(3, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals(4, jobStatus.getIdsByIndexingStatus(IndexingStatus.SKIP).size());
+ }
+
+ @Test
+ public void should_returnValidResponse_finalizeRecordStatusTest() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"create\"}," +
+ "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"create\"}," +
+ "{\"id\":\"tenant1:doc:test3\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:test4\",\"kind\":\"tenant1:testindexer12:well:1.0.0\",\"op\":\"update\"}]";
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = (new Gson()).fromJson(recordChangedMessages, listType);
+ this.sut.initialize(recordInfos);
+
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test1", IndexingStatus.PROCESSING, 1, "");
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test2", IndexingStatus.SUCCESS, 200, "");
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test3", IndexingStatus.FAIL, 500, "");
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test4", IndexingStatus.SKIP, 404, "");
+
+ String finalizeMessage = "unit test force fail";
+ this.sut.finalizeRecordStatus(finalizeMessage);
+ assertEquals(0, this.sut.getIdsByIndexingStatus(IndexingStatus.PROCESSING).size());
+ assertEquals(1, this.sut.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+ assertEquals(2, this.sut.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals(1, this.sut.getIdsByIndexingStatus(IndexingStatus.SKIP).size());
+ assertEquals(finalizeMessage, this.sut.getStatusesList().get(0).getLatestTrace());
+ }
+
+ private JobStatus insertTestCasesIntoJobStatus() {
+ JobStatus jobStatus = new JobStatus();
+ jobStatus.addOrUpdateRecordStatus("processing1", IndexingStatus.PROCESSING, 1, "");
+ jobStatus.addOrUpdateRecordStatus("success1", IndexingStatus.SUCCESS, 1, "");
+ jobStatus.addOrUpdateRecordStatus("success2", IndexingStatus.SUCCESS, 1, "");
+ jobStatus.addOrUpdateRecordStatus("fail1", IndexingStatus.FAIL, 1, "");
+ jobStatus.addOrUpdateRecordStatus("fail2", IndexingStatus.FAIL, 1, "");
+ jobStatus.addOrUpdateRecordStatus("fail3", IndexingStatus.FAIL, 1, "");
+ jobStatus.addOrUpdateRecordStatus("skipped1", IndexingStatus.SKIP, 1, "");
+ jobStatus.addOrUpdateRecordStatus("skipped2", IndexingStatus.SKIP, 1, "");
+ jobStatus.addOrUpdateRecordStatus("skipped3", IndexingStatus.SKIP, 1, "");
+ jobStatus.addOrUpdateRecordStatus("skipped4", IndexingStatus.SKIP, 1, "");
+ jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS);
+ return jobStatus;
+ }
+
+ @Test
+ public void should_returnNullList_getRecordStatuses() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer12:well:1.0.0\",\"op\":\"create\"}]";
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = (new Gson()).fromJson(recordChangedMessages, listType);
+ this.sut.initialize(recordInfos);
+
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test1", IndexingStatus.SUCCESS, 1, "");
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test2", IndexingStatus.SUCCESS, 1, "");
+
+ List<RecordStatus> statuses = this.sut.getRecordStatuses(IndexingStatus.FAIL, OperationType.create);
+ assertTrue(statuses.isEmpty());
+ assertEquals(statuses.size(), 0);
+ }
+
+ @Test
+ public void should_returnValidList_getRecordStatuses() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer12:well:1.0.0\",\"op\":\"create\"}]";
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> recordInfos = (new Gson()).fromJson(recordChangedMessages, listType);
+ this.sut.initialize(recordInfos);
+
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test1", IndexingStatus.SUCCESS, 1, "");
+ this.sut.addOrUpdateRecordStatus("tenant1:doc:test2", IndexingStatus.SUCCESS, 1, "");
+
+ List<RecordStatus> statuses = this.sut.getRecordStatuses(IndexingStatus.SUCCESS, OperationType.create);
+ assertNotNull(statuses);
+ assertEquals(1, statuses.size());
+
+ List<String> toString = statuses.stream().map(RecordStatus::toString).collect(Collectors.toList());
+ assertEquals(toString.get(0), "RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer12:well:1.0.0, operationType=create, status=SUCCESS)");
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/util/RecordInfoTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/util/RecordInfoTest.java
new file mode 100644
index 000000000..7083fca19
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/util/RecordInfoTest.java
@@ -0,0 +1,102 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opendes.core.util.AppException;
+import org.opendes.indexer.model.OperationType;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.fail;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class RecordInfoTest {
+
+ private List<RecordInfo> msgs;
+ private final String recordChangedMessages = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:test4\",\"kind\":\"tenant1:testindexer4:well:1.0.0\",\"op\":\"delete\"}," +
+ "{\"id\":\"tenant1:doc:test5\",\"kind\":\"tenant1:testindexer4:well:1.0.0\",\"op\":\"delete\"}," +
+ "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}," +
+ "{\"id\":\"tenant1:doc:test3\",\"kind\":\"tenant1:testindexer3:well:1.0.0\",\"op\":\"update\"}]";
+
+ @Before
+ public void setup() {
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ }
+
+ @Test
+ public void should_returnOpMap_given_createOrUpdateOperations_getUpsertRecordIdsTest() {
+ Map<String, Map<String, OperationType>> kindRecordOpMap = RecordInfo.getUpsertRecordIds(msgs);
+ Assert.assertEquals(2, kindRecordOpMap.size());
+ Assert.assertEquals(OperationType.create, kindRecordOpMap.get("tenant1:testindexer2:well:1.0.0").get("tenant1:doc:test2"));
+ Assert.assertEquals(OperationType.update, kindRecordOpMap.get("tenant1:testindexer3:well:1.0.0").get("tenant1:doc:test3"));
+ }
+
+ @Test
+ public void should_return400_getUpsertRecordTest(List<RecordInfo> msgs, String errorMessage) {
+ try {
+ RecordInfo.getUpsertRecordIds(msgs);
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode());
+ Assert.assertEquals(errorMessage, e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return400_given_nullMessage_getUpsertRecordTest() {
+ should_return400_getUpsertRecordTest(null, "Error parsing upsert records in request payload.");
+ }
+
+ @Test
+ public void should_returnValidResponse_getDeleteRecordIdsTest() {
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(msgs);
+ Assert.assertEquals(2, deleteRecordMap.size());
+ Assert.assertEquals(1, deleteRecordMap.get("tenant1:testindexer1:well:1.0.0").size());
+ Assert.assertEquals(2, deleteRecordMap.get("tenant1:testindexer4:well:1.0.0").size());
+ }
+
+ @Test
+ public void should_return400_getDeleteRecordTest(List<RecordInfo> msgs, String errorMessage) {
+ try {
+ RecordInfo.getDeleteRecordIds(msgs);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode());
+ Assert.assertEquals(errorMessage, e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return400_givenNullMessage_getDeleteRecordIdsTest() {
+ should_return400_getDeleteRecordTest(null, "Error parsing delete records in request payload.");
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/DateTimeParserTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/DateTimeParserTest.java
new file mode 100644
index 000000000..cd6140d31
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/DateTimeParserTest.java
@@ -0,0 +1,92 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util.parser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+@RunWith(SpringRunner.class)
+public class DateTimeParserTest {
+ @InjectMocks
+ private DateTimeParser sut;
+
+ @Test
+ public void should_returnCorrectUtc_given_validDateFormat_convertDateObjectToUtcTest() {
+ // parse "yyyy-MM-dd HH:mm:ss"
+ assertEquals("2000-01-02T10:10:44+0000", this.sut.convertDateObjectToUtc("2000-01-02 10:10:44"));
+
+ // test "yyyy-MM-dd'T'HH:mm:ss"
+ assertEquals("2000-01-02T10:10:44+0000", this.sut.convertDateObjectToUtc("2000-01-02T10:10:44"));
+
+ // parse "yyyy-MM-dd HH:mm:ss.SSS"
+ assertEquals("2000-01-02T10:10:44.123+0000", this.sut.convertDateObjectToUtc("2000-01-02 10:10:44.123"));
+
+ // parse "yyyy-MM-dd HH:mm:ss.SSSSSS"
+ assertEquals("2000-01-02T10:10:44.123+0000", this.sut.convertDateObjectToUtc("2000-01-02 10:10:44.123000"));
+
+ // parse "yyyy-MM-dd HH:mm:ss.SSSSSS"
+ assertEquals("2000-01-02T10:10:44.123+0000", this.sut.convertDateObjectToUtc("2000-01-02 10:10:44.1230000"));
+
+ // parse "yyyy-MM-dd"
+ assertEquals("2000-01-02T00:00:00+0000", this.sut.convertDateObjectToUtc("2000-01-02"));
+ assertEquals("0001-01-01T00:00:00+0000", this.sut.convertDateObjectToUtc("0001-01-01"));
+
+ // parse "yyyyMMdd"
+ assertEquals("2000-01-02T00:00:00+0000", this.sut.convertDateObjectToUtc("20000102"));
+
+ // parse string with zulu indicator: "yyyy-MM-dd'T'HH:mm:ssz"
+ assertEquals("2018-11-06T19:37:11.128+0000", this.sut.convertDateObjectToUtc("2018-11-06T19:37:11.128Z"));
+
+ // parse with offset indicator: "yyyy-MM-dd HH:mm:ss.SSSXXX"
+ assertEquals("1968-11-01T00:00:00+0000", this.sut.convertDateObjectToUtc("1968-11-01T00:00:00+00:00"));
+ assertEquals("1968-11-01T00:00:45.56+0000", this.sut.convertDateObjectToUtc("1968-11-01T00:00:45.56+00:00"));
+
+ // parse string with offset: "yyyy-MM-dd'T'HH:mm:ssXXX"
+ assertEquals("2000-01-02T10:10:44-0830", this.sut.convertDateObjectToUtc("2000-01-02T10:10:44-08:30"));
+
+ // parse "EEE MMM dd HH:mm:ss zzz yyyy"
+ assertEquals("2018-07-18T10:10:44+0000", this.sut.convertDateObjectToUtc("Wed Jul 18 10:10:44 PST 2018"));
+ }
+
+ @Test
+ public void should_returnNull_given_emptyOrNull_covertDateObjectToUtcTest() {
+ assertNull(this.sut.convertDateObjectToUtc(""));
+ assertNull(this.sut.convertDateObjectToUtc(null));
+ }
+
+ @Test
+ public void should_returnNull_given_invalidYearMonthDay_covertDateObjectToUtcTest() {
+ assertNull(this.sut.convertDateObjectToUtc("2000-00-02 10:10:44.123"));
+ assertNull(this.sut.convertDateObjectToUtc("2000-01-40 10:10:44.123"));
+ assertNull(this.sut.convertDateObjectToUtc("3000-14-02 10:10:44.123"));
+ }
+
+ @Test
+ public void should_returnNull_given_invalidDateFormat_convertDateObjectToUtcTest() {
+ assertNull(this.sut.convertDateObjectToUtc("07/01/2010"));
+ }
+
+ @Test
+ public void should_returnNull_given_invalidDate_convertDateObjectToUtcTest() {
+ assertNull(this.sut.convertDateObjectToUtc("N/A"));
+ assertNull(this.sut.convertDateObjectToUtc(".2190851121908511EE44"));
+ assertNull(this.sut.convertDateObjectToUtc("E.2131"));
+ }
+}
diff --git a/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/NumberParserTest.java b/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/NumberParserTest.java
new file mode 100644
index 000000000..979d2d3c7
--- /dev/null
+++ b/indexer-service-root/src/test/java/org/opendes/indexer/util/parser/NumberParserTest.java
@@ -0,0 +1,205 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.util.parser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.BiConsumer;
+
+import static org.junit.matchers.JUnitMatchers.*;
+import static org.junit.Assert.*;
+
+@RunWith(SpringRunner.class)
+public class NumberParserTest {
+
+ @InjectMocks
+ private NumberParser sut;
+
+ @Test
+ public void should_parseInteger() {
+ int result = this.sut.parseInteger("lat", "");
+ assertEquals(result, 0);
+
+ result = this.sut.parseInteger("lat", null);
+ assertEquals(result, 0);
+
+ result = this.sut.parseInteger("lat", "101959.1");
+ assertEquals(result, 101959);
+
+ result = this.sut.parseInteger("lat", 139);
+ assertEquals(result, 139);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidInteger() {
+ final List<Object> outOfRangeInputs = Arrays.asList(
+ "2147483648",
+ 2147483648L,
+ -2147483649L
+ );
+ this.validateInput(this.sut::parseInteger, outOfRangeInputs, "int", "number parsing error, integer out of range: attribute: %s | value: %s");
+
+ final List<Object> invalidInputs = Arrays.asList(
+ "garbage",
+ "135.5ga"
+ );
+ this.validateInput(this.sut::parseInteger, invalidInputs, "int", "number parsing error: attribute: %s | value: %s");
+ }
+
+ @Test
+ public void should_parseLong() {
+ long result = this.sut.parseLong("lat", "");
+ assertEquals(result, 0);
+
+ result = this.sut.parseLong("lat", null);
+ assertEquals(result, 0);
+
+ result = this.sut.parseLong("lat", 4115420654264075766L);
+ assertEquals(result, 4115420654264075766L);
+
+ result = this.sut.parseLong("lat", "4115420");
+ assertEquals(result, 4115420);
+
+// result = this.sut.parseLong("lat", "4L");
+// assertEquals(result, 4L);
+
+ result = this.sut.parseLong("lat", 4L);
+ assertEquals(result, 4L);
+
+ result = this.sut.parseLong("lat", 139);
+ assertEquals(result, 139);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidLong() {
+ final List<Object> outOfRangeInputs = Arrays.asList(
+ "9223372036854775808",
+ new BigInteger("9223372036854775808"),
+ new BigInteger("-9223372036854775809")
+ );
+ this.validateInput(this.sut::parseLong, outOfRangeInputs, "long", " is out of range for a long");
+
+ final List<Object> invalidInputs = Arrays.asList(
+ "garbage",
+ "135.5ga"
+ );
+ this.validateInput(this.sut::parseLong, invalidInputs, "long", "number parsing error: attribute: %s | value: %s");
+ }
+
+ @Test
+ public void should_parseFloat() {
+ float result = this.sut.parseFloat("lon", "");
+ assertEquals(result, 0, 0);
+
+ result = this.sut.parseFloat("lon", null);
+ assertEquals(result, 0, 0);
+
+ result = this.sut.parseFloat("lon", "101959.1");
+ assertEquals(result, 101959.1, 0.01);
+
+ result = this.sut.parseFloat("lon", "1.1f");
+ assertEquals(result, 1.1, 0.01);
+
+ result = this.sut.parseFloat("lon", 1.1f);
+ assertEquals(result, 1.1, 0.01);
+
+ result = this.sut.parseFloat("lon", 1.1);
+ assertEquals(result, 1.1, 0.01);
+
+ result = this.sut.parseFloat("lon", 139);
+ assertEquals(result, 139, 0);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidFloat() {
+ final List<Object> outOfRangeInputs = Arrays.asList(
+ "3.4028235E39",
+ 3.4028235E39d,
+ -3.4028235E39d
+ );
+ this.validateInput(this.sut::parseFloat, outOfRangeInputs, "float", "number parsing error, float only supports finite values: attribute: %s | value: %s");
+
+ final List<Object> invalidInputs = Arrays.asList(
+ "garbage",
+ "135.5ga",
+ Float.NaN,
+ Float.POSITIVE_INFINITY,
+ Float.NEGATIVE_INFINITY
+ );
+ this.validateInput(this.sut::parseFloat, invalidInputs, "float", "number parsing error: attribute: %s | value: %s");
+ }
+
+ @Test
+ public void should_parseDouble() {
+ double result = this.sut.parseDouble("location", "");
+ assertEquals(result, 0, 0);
+
+ result = this.sut.parseDouble("location", null);
+ assertEquals(result, 0, 0);
+
+ result = this.sut.parseDouble("location", "20.0");
+ assertEquals(result, 20.0, 0.01);
+
+ result = this.sut.parseDouble("location", "20.0d");
+ assertEquals(result, 20.0, 0.01);
+
+ result = this.sut.parseDouble("location", 20.0d);
+ assertEquals(result, 20.0, 0.01);
+
+ result = this.sut.parseDouble("location", 1.1);
+ assertEquals(result, 1.1, 0);
+
+ result = this.sut.parseDouble("location", 139);
+ assertEquals(result, 139, 0);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidDouble() {
+ final List<Object> outOfRangeInputs = Arrays.asList(
+ "1.7976931348623157E309",
+ new BigDecimal("1.7976931348623157E309"),
+ new BigDecimal("-1.7976931348623157E309")
+ );
+ this.validateInput(this.sut::parseDouble, outOfRangeInputs, "double", "number parsing error, double only supports finite values: attribute: %s | value: %s");
+
+ final List<Object> invalidInputs = Arrays.asList(
+ "garbage",
+ "135.5ga",
+ Double.NaN,
+ Double.POSITIVE_INFINITY,
+ Double.NEGATIVE_INFINITY
+ );
+ this.validateInput(this.sut::parseDouble, invalidInputs, "double", "number parsing error: attribute: %s | value: %s");
+ }
+
+ private void validateInput(BiConsumer<String, Object> parser, List<Object> inputs, String type, String errorMessage) {
+ for (Object attributeVal : inputs) {
+ try {
+ parser.accept("dummyAttribute", attributeVal);
+ fail(String.format("Parsing exception expected for %s with value [ %s ]", type, attributeVal));
+ } catch (IllegalArgumentException e) {
+ assertThat(String.format("Incorrect error message for %s with value [ %s ]", type, attributeVal),
+ e.getMessage(), containsString(String.format(errorMessage, "dummyAttribute", attributeVal)));
+ }
+ }
+ }
+}
diff --git a/indexer-service-root/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/indexer-service-root/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 000000000..ca6ee9cea
--- /dev/null
+++ b/indexer-service-root/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index a1da162b4..151d85522 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,14 +4,166 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.slb</groupId>
- <artifactId>dps-indexer-boot-mvn</artifactId>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.1.7.RELEASE</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+
+ <groupId>org.opendes.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
<version>1.0-SNAPSHOT</version>
- <modules>
- <module>dps-indexer</module>
- <module>dps-indexer-queue</module>
- </modules>
+ <description>Indexer Service</description>
+<!-- <modules>-->
+<!-- <module>indexer-service-root</module>-->
+<!-- <module>indexer-service-gcp</module>-->
+<!-- <module>indexer-service-azure</module>-->
+<!-- </modules>-->
+
+ <properties>
+ <java.version>1.8</java.version>
+ <springfox-version>2.7.0</springfox-version>
+ <spring-cloud.version>Greenwich.SR2</spring-cloud.version>
+<!-- <java.version>1.8</java.version>-->
+<!-- <maven.compiler.target>1.8</maven.compiler.target>-->
+<!-- <maven.compiler.source>1.8</maven.compiler.source>-->
+<!-- <maven.war.plugin>2.6</maven.war.plugin>-->
+<!-- <appengine.maven.plugin>1.0.0</appengine.maven.plugin>-->
+<!-- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>-->
+<!-- <failOnMissingWebXml>false</failOnMissingWebXml>-->
+<!-- <spring-cloud.version>Greenwich.SR2</spring-cloud.version>-->
+<!-- <springfox-version>2.7.0</springfox-version>-->
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>dev-azure-com-slb-des-ext-collaboration-os-core</id>
+ <url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
+ <!-- Test Dependencies -->
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.7</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>root</id>
+ <activation>
+ <!-- this profile is active by default -->
+ <activeByDefault>true</activeByDefault>
+ </activation>
+
+ <modules>
+ <module>indexer-service-root</module>
+ </modules>
+ </profile>
+
+ <profile>
+ <id>gcp</id>
+ <activation>
+ <property>
+ <name>vendor</name>
+ <value>gcp</value>
+ </property>
+ </activation>
+
+ <modules>
+ <module>indexer-service-gcp</module>
+ </modules>
+
+ </profile>
+ <profile>
+ <id>azure</id>
+ <activation>
+ <property>
+ <name>vendor</name>
+ <value>azure</value>
+ </property>
+ </activation>
+
+ <modules>
+ <module>indexer-service-azure</module>
+ </modules>
+
+ </profile>
+<!-- <!– Unit test only –>-->
+<!-- <profile>-->
+<!-- <id>dev</id>-->
+<!-- <properties>-->
+<!-- <skip.unit.tests>false</skip.unit.tests>-->
+<!-- <skip.integration.tests>true</skip.integration.tests>-->
+<!-- </properties>-->
+<!-- </profile>-->
+<!-- <!– Integration test only –>-->
+<!-- <profile>-->
+<!-- <id>integration-test</id>-->
+<!-- <properties>-->
+<!-- <skip.integration.tests>false</skip.integration.tests>-->
+<!-- <skip.unit.tests>true</skip.unit.tests>-->
+<!-- </properties>-->
+<!-- </profile>-->
+<!-- <!– Unit and Integration tests combined –>-->
+<!-- <profile>-->
+<!-- <id>test-all</id>-->
+<!-- <properties>-->
+<!-- <skip.integration.tests>false</skip.integration.tests>-->
+<!-- <skip.unit.tests>false</skip.unit.tests>-->
+<!-- </properties>-->
+<!-- </profile>-->
+ </profiles>
</project>
\ No newline at end of file
--
GitLab