From 57b878344cf74434a4d18a875419b58a60ab4c7c Mon Sep 17 00:00:00 2001
From: NThakur4 <nthakur4@slb.com>
Date: Thu, 14 Apr 2022 20:31:11 -0500
Subject: [PATCH] doc and refactor

---
 docs/tutorial/IndexerService.md               | 100 ++++++++++++++----
 .../osdu/indexer/service/IMappingService.java |   2 +-
 .../service/IndexerMappingServiceImpl.java    |   8 +-
 .../indexer/service/IndexerServiceImpl.java   |   2 +-
 .../service/IndexerMappingServiceTest.java    |  10 +-
 .../indexer/service/IndexerServiceImpl.java   |   2 +-
 6 files changed, 98 insertions(+), 26 deletions(-)

diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md
index 5208cc9b3..f18ad0cbe 100644
--- a/docs/tutorial/IndexerService.md
+++ b/docs/tutorial/IndexerService.md
@@ -5,9 +5,11 @@
 - [Indexer service](#indexer-service)
 - [Introduction](#introduction)
 - [Indexer API access](#indexer-api-access)
-- [Version info endpoint](#version-info-endpoint)
-- [Reindex](#reindex)
-- [Data Partition provision](#data-partition-provision)
+- [API Reference](#api-reference)
+  - [Version info endpoint](#version-info-endpoint)
+  - [Reindex](#reindex)
+  - [Data Partition provision](#data-partition-provision)
+  - [Schema change](#schema-change)
 - [Schema Service adoption](#schema-service-adoption)
     * [R3 Schema Support](#r3-schema-support)
 - [Troubleshoot Indexing Issues](#troubleshoot-indexing-issues)  
@@ -51,9 +53,17 @@ If the service is initiating the request, an ID should be generated. If the `cor
 
 [Back to table of contents](#TOC)
 
-## Version info endpoint
+## API Reference
 
-For deployment available public `/info` endpoint, which provides build and git related information.
+### Version info endpoint
+
+Provides build and git related information.
+
+#### Request
+
+```http request
+GET /api/indexer/v2/info HTTP/1.1
+```
 
 #### Example response:
 
@@ -86,20 +96,16 @@ This endpoint takes information from files, generated by `spring-boot-maven-plug
 
 [Back to table of contents](#TOC)
 
-## Reindex <a name="reindex"></a>
+### Reindex <a name="reindex"></a>
 
-Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an
-asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
-appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making
-query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
+Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
+appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
 multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc.
 
-__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes
-during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API
-will use the same schema and overwrite records with the same ids.
+#### Request
 
-```
-POST /api/indexer/v2/reindex
+```http request
+POST /api/indexer/v2/reindex HTTP/1.1
 {
   "kind": "opendes:welldb:wellbore:1.0.0"
 }
@@ -121,14 +127,30 @@ curl --request POST \
 
 </details>
 
+#### Prerequisite
+
+Users must be a member of `users.datalake.admins` or `users.datalake.ops` group.
+
+#### Query parameters
+
+`force_clean` <br />
+&emsp;&emsp;(optional, Boolean) If a kind has been previously indexed with a schema and if you wish to apply latest schema changes before re-indexing, than use this query parameter. It will drop the current Index schema, apply latest schema changes & re-index records. If `false`, reindex API
+will use the same schema and overwrite records with the same ids. Default value is `false`.
+
+#### Request body
+
+`kind` <br />
+&emsp;&emsp;(required, String) Kind to be re-indexed. 
+
+
 [Back to table of contents](#TOC)
 
-## Data Partition provision <a name="data-partition-provision"></a>
+### Data Partition provision <a name="data-partition-provision"></a>
 
 Configures Search backend for a data partition. 
 
-```
-PUT /api/indexer/v2/partitions/provision
+```http request 
+PUT /api/indexer/v2/partitions/provision HTTP/1.1
 ```
 
 <details><summary>**Curl**</summary>
@@ -143,10 +165,52 @@ curl --request PUT \
 ```
 </details>
 
+#### Prerequisite
+
+Users must be a member of `users.datalake.ops` group.
+
 > __NOTE__: API should be run at-least once at the data partition provisioning to configure required resources/settings.
 
 [Back to table of contents](#TOC)
 
+### Schema change <a name="schema-change"></a>
+
+Schema change event listener endpoint.
+
+> __Note:__ This is internal API and shouldn't be exposed publicly.
+
+#### Request
+
+```http request
+POST /api/indexer/v2/_dps/task-handlers/schema-worker HTTP/1.1
+{
+    "messageId": "676894654",
+    "publishTime": "2017-03-19T00:00:00",
+    "attributes": {
+        "data-partition-id": "opendes",
+        "correlation-id": "b5a281bd-f59d-4db2-9939-b2d85036fc7e"
+    },
+    "data": "[{\"kind\":\"slb:indexer:test-data--SchemaEventIntegration:1.0.0\",\"op\":\"create\"}]"
+}
+```
+
+#### Request body
+
+`messageId` <br />
+&emsp;&emsp;(optional, String) Event message id.
+
+`publishTime` <br />
+&emsp;&emsp;(optional, String) Event publish time.
+
+`attributes.data-partition-id` <br />
+&emsp;&emsp;(required, String) Data partition id for which this message is targeted.
+
+`attributes.correlation-id` <br />
+&emsp;&emsp;(optional, String) Correlation-id to enable tracing.
+
+`data` <br />
+&emsp;&emsp;(required, String) Schema change event message json string. Only `create` and `update` events are supported.
+
 ## Schema Service adoption <a name="schema-service-adoption"></a>
 
 Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. The Indexer
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
index 8c89e4e64..f0009bd04 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
@@ -33,5 +33,5 @@ public interface IMappingService {
 
     void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception;
 
-    void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index, String kind) throws Exception;
+    void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception;
 }
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 8df970bb7..6b0b6899c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -36,6 +36,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
 import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
 import org.opengroup.osdu.core.common.search.Preconditions;
 import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
 import org.opengroup.osdu.indexer.util.ElasticClientHandler;
@@ -59,6 +60,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
     private PartitionSafeIndexCache indexCache;
     @Autowired
     private IMappingService mappingService;
+    @Autowired
+    private ElasticIndexNameResolver elasticIndexNameResolver;
 
     private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
 
@@ -173,7 +176,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
     }
 
     @Override
-    public void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index, String kind) throws Exception {
+    public void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception {
+        String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
         final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
 
         try {
@@ -206,7 +210,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
         }
 
         Map<String, Object> properties = new HashMap<>();
-        String[] parts = kind.split(":");
+        String[] parts = schema.getKind().split(":");
         String authority = parts[0];
         String source = parts[1];
         for (String attribute : missing) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 3f659080c..6bec8964f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -367,7 +367,7 @@ public class IndexerServiceImpl implements IndexerService {
 
             // check if index exist and sync meta attribute schema if required
             if (this.indicesService.isIndexReady(restClient, index)) {
-                this.mappingService.syncIndexMappingIfRequired(restClient, index, schema.getKind());
+                this.mappingService.syncIndexMappingIfRequired(restClient, schema);
                 continue;
             }
 
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index 0d959f1eb..ec1f61e4b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -27,6 +27,7 @@ import org.mockito.Mock;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
 import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
 import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
 import org.opengroup.osdu.indexer.util.ElasticClientHandler;
 import org.opengroup.osdu.indexer.util.TypeMapper;
@@ -73,6 +74,8 @@ public class IndexerMappingServiceTest {
     private PartitionSafeIndexCache indexCache;
     @Mock
     private IMappingService mappingService;
+    @Mock
+    private ElasticIndexNameResolver elasticIndexNameResolver;
     @InjectMocks
     private IndexerMappingServiceImpl sut;
 
@@ -89,6 +92,7 @@ public class IndexerMappingServiceTest {
         this.indicesClient = PowerMockito.mock(IndicesClient.class);
         this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
 
+        when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(index);
         when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
         when(this.restClient.performRequest(any())).thenReturn(response);
         when(this.response.getStatusLine()).thenReturn(statusLine);
@@ -178,7 +182,7 @@ public class IndexerMappingServiceTest {
         final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
         when(this.indexCache.get(cacheKey)).thenReturn(true);
 
-        this.sut.syncIndexMappingIfRequired(restHighLevelClient, index, kind);
+        this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
 
         verifyNoMoreInteractions(this.mappingService);
     }
@@ -189,7 +193,7 @@ public class IndexerMappingServiceTest {
         final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"opendes\"},\"createTime\":{\"type\":\"date\"},\"createUser\":{\"type\":\"keyword\"},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"modifyTime\":{\"type\":\"date\"},\"modifyUser\":{\"type\":\"keyword\"},\"namespace\":{\"type\":\"keyword\"},\"source\":{\"type\":\"constant_keyword\",\"value\":\"test\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
         when(this.mappingService.getIndexMapping(restHighLevelClient, index)).thenReturn(mapping);
 
-        this.sut.syncIndexMappingIfRequired(restHighLevelClient, index, kind);
+        this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
 
         verify(this.indexCache, times(1)).get(cacheKey);
         verify(this.indexCache, times(1)).put(cacheKey, true);
@@ -205,7 +209,7 @@ public class IndexerMappingServiceTest {
         doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
         doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class), any(RequestOptions.class));
 
-        this.sut.syncIndexMappingIfRequired(restHighLevelClient, index, kind);
+        this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
 
         verify(this.indexCache, times(1)).get(cacheKey);
         verify(this.indexCache, times(1)).put(cacheKey, true);
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index d6d79ac87..505180648 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -425,7 +425,7 @@ public class IndexerServiceImpl implements IndexerService {
 
       // check if index exist and sync meta attribute schema if required
       if (this.indicesService.isIndexExist(restClient, index)) {
-        this.mappingService.syncIndexMappingIfRequired(restClient, index, schema.getKind());
+        this.mappingService.syncIndexMappingIfRequired(restClient, schema);
         continue;
       }
 
-- 
GitLab