diff --git a/.gitignore b/.gitignore
index d5f56ddd691f1b95cd115b7425d0d96918a0c339..81a6ed9f80ee2d3a7a930b81fa1176651f20c0ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ target/
!**/src/main/**
!**/src/test/**
+.DS_Store
### STS ###
.apt_generated
.classpath
@@ -35,3 +36,4 @@ build/
target/*
*/target
/mvn
+provider/indexer-gcp/bin/
\ No newline at end of file
diff --git a/.idea/$PRODUCT_WORKSPACE_FILE$ b/.idea/$PRODUCT_WORKSPACE_FILE$
deleted file mode 100644
index 3733e0d369c0331583e53353e77299dc56783aab..0000000000000000000000000000000000000000
--- a/.idea/$PRODUCT_WORKSPACE_FILE$
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
- <component name="masterDetails">
- <states>
- <state key="ProjectJDKs.UI">
- <settings>
- <last-edited>1.8</last-edited>
- <splitter-proportions>
- <option name="proportions">
- <list>
- <option value="0.2" />
- </list>
- </option>
- </splitter-proportions>
- </settings>
- </state>
- </states>
- </component>
-</project>
\ No newline at end of file
diff --git a/README.md b/README.md
index 0ca446aab9d09eac8625b53e3df8da661976c458..e37e4b10b321079b915014153dd4c17725e8c7d4 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,20 @@
-# Introduction
-TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project.
-
-# Getting Started
-TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
-1. Installation process
-2. Software dependencies
-3. Latest releases
-4. API references
-
-# Build and Test
-TODO: Describe and show how to build your code and run the tests.
-
-# Contribute
-TODO: Explain how other users and developers can contribute to make your code better.
-
-If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
-- [ASP.NET Core](https://github.com/aspnet/Home)
-- [Visual Studio Code](https://github.com/Microsoft/vscode)
+# Introduction
+TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project.
+
+# Getting Started
+TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
+1. Installation process
+2. Software dependencies
+3. Latest releases
+4. API references
+
+# Build and Test
+TODO: Describe and show how to build your code and run the tests.
+
+# Contribute
+TODO: Explain how other users and developers can contribute to make your code better.
+
+If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
+- [ASP.NET Core](https://github.com/aspnet/Home)
+- [Visual Studio Code](https://github.com/Microsoft/vscode)
- [Chakra Core](https://github.com/Microsoft/ChakraCore)
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..3b33fd2ebb76f058d9f4a41120383d1c48d1b738
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
@@ -0,0 +1,20 @@
+package org.opengroup.osdu.indexer;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthIndicatorAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ComponentScan({"org.opengroup.osdu.core.common","org.opengroup.osdu.indexer", "org.opengroup.osdu.is"})
+@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+public class IndexerApplication {
+ public static void main( String[] args )
+ {
+ SpringApplication.run(IndexerApplication.class, args);
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/ServerletInitializer.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/ServerletInitializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..b29ac79f424e29b1b0e9f85a0cab44c464fe4db2
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/ServerletInitializer.java
@@ -0,0 +1,11 @@
+package org.opengroup.osdu.indexer;
+
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+
+public class ServerletInitializer extends SpringBootServletInitializer {
+ @Override
+ protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
+ return application.sources(IndexerApplication.class);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index 6ac08b6ee75a72b5a63f1e21d70b8d370e8601c7..545570a22505e968ce7070578f5a978cb43d73fc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -20,15 +20,15 @@ import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import io.swagger.annotations.ApiOperation;
import lombok.extern.java.Log;
-import org.opengroup.osdu.core.api.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
import org.opengroup.osdu.indexer.SwaggerDoc;
-import org.opengroup.osdu.indexer.util.JobStatus;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.indexer.util.RecordInfo;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index d5cbe7ecc60f5af57c63bdfe5dac64c0a2c65937..b4a7f30a05db4ad8d2422698eb582c61b8906f6c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -14,10 +14,10 @@
package org.opengroup.osdu.indexer.api;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.coreis.SearchServiceRole;
+import org.opengroup.osdu.core.common.model.indexer.AuditLogger;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.is.core.model.SearchServiceRole;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PostMapping;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlements.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlements.java
index eea65eb172cee269ea810ceba7faa13aebea91cd..45dc213c4a6a35d7a605452b9a9436eced189ee6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlements.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlements.java
@@ -14,19 +14,18 @@
package org.opengroup.osdu.indexer.auth;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.api.entitlements.EntitlementsException;
-import org.opengroup.osdu.core.api.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.api.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.api.entitlements.models.GroupInfo;
-import org.opengroup.osdu.core.api.entitlements.models.Groups;
-import org.opengroup.osdu.core.httpclient.HttpResponse;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.AuthorizationResponse;
-import org.opengroup.osdu.is.core.provider.interfaces.auth.AuthorizationService;
-import org.opengroup.osdu.is.core.util.AppException;
-import org.opengroup.osdu.is.core.util.HeadersUtil;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.entitlements.EntitlementsException;
+import org.opengroup.osdu.core.common.model.core.entitlements.GroupInfo;
+import org.opengroup.osdu.core.common.model.core.entitlements.Groups;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AuthorizationResponse;
+import org.opengroup.osdu.core.common.service.core.HttpResponse;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+import org.opengroup.osdu.core.common.service.coreis.HeadersUtil;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.AuthorizationService;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
deleted file mode 100644
index 98af0f880809d0d02ff15ce3fa7bdd7e092fec9b..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
+++ /dev/null
@@ -1,220 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.logging;
-
-import com.google.common.base.Strings;
-import org.opengroup.osdu.core.logging.payload.AuditAction;
-import org.opengroup.osdu.core.logging.payload.AuditPayload;
-import org.opengroup.osdu.core.logging.payload.AuditStatus;
-
-import java.util.List;
-
-import static org.opengroup.osdu.core.logging.payload.AuditPayload.builder;
-
-public class AuditEvents {
-
- private static final String INDEX_CREATE_RECORD_ACTION_ID = "IN001";
- private static final String INDEX_CREATE_RECORDS_SUCCESS = "Successfully created record in index";
- private static final String INDEX_CREATE_RECORDS_FAILURE = "Failed creating record in index";
-
- private static final String INDEX_UPDATE_RECORD_ACTION_ID = "IN002";
- private static final String INDEX_UPDATE_RECORDS_SUCCESS = "Successfully updated record in index";
- private static final String INDEX_UPDATE_RECORDS_FAILURE = "Failed updating record in index";
-
- private static final String INDEX_DELETE_RECORD_ACTION_ID = "IN003";
- private static final String INDEX_DELETE_RECORDS_SUCCESS = "Successfully deleted record in index";
- private static final String INDEX_DELETE_RECORDS_FAILURE = "Failed deleting record in index";
-
- private static final String INDEX_PURGE_RECORD_ACTION_ID = "IN004";
-
- private static final String REINDEX_KIND_ACTION_ID = "IN007";
- private static final String REINDEX_KIND_OPERATION = "Reindex kind";
-
- private static final String COPY_INDEX_ACTION_ID = "IN008";
- private static final String COPY_INDEX_OPERATION = "Copy index";
-
- private static final String GET_TASK_STATUS_ACTION_ID = "IN009";
- private static final String GET_TASK_STATUS_OPERATION = "Get task status";
-
- private static final String RUN_JOB_ACTION_ID = "IN010";
- private static final String RUN_JOB_MESSAGE_SUCCESS = "Index clean-up status job run success";
-
- private static final String INDEX_MAPPING_UPDATE_ACTION_ID = "IN0011";
- private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully updated index mapping";
- private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed updating index mapping";
-
- private final String user;
-
- public AuditEvents(String user) {
- if (Strings.isNullOrEmpty(user)) {
- throw new IllegalArgumentException("User not provided for audit events.");
- }
- this.user = user;
- }
-
- public AuditPayload getIndexCreateRecordSuccessEvent(List<String> resources) {
- return builder()
- .action(AuditAction.CREATE)
- .status(AuditStatus.SUCCESS)
- .actionId(INDEX_CREATE_RECORD_ACTION_ID)
- .message(INDEX_CREATE_RECORDS_SUCCESS)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexCreateRecordFailEvent(List<String> resources) {
- return builder()
- .action(AuditAction.CREATE)
- .status(AuditStatus.FAILURE)
- .actionId(INDEX_CREATE_RECORD_ACTION_ID)
- .message(INDEX_CREATE_RECORDS_FAILURE)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexUpdateRecordSuccessEvent(List<String> resources) {
- return builder()
- .action(AuditAction.UPDATE)
- .status(AuditStatus.SUCCESS)
- .actionId(INDEX_UPDATE_RECORD_ACTION_ID)
- .message(INDEX_UPDATE_RECORDS_SUCCESS)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexUpdateRecordFailEvent(List<String> resources) {
- return builder()
- .action(AuditAction.UPDATE)
- .status(AuditStatus.FAILURE)
- .actionId(INDEX_UPDATE_RECORD_ACTION_ID)
- .message(INDEX_UPDATE_RECORDS_FAILURE)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexDeleteRecordSuccessEvent(List<String> resources) {
- return builder()
- .action(AuditAction.DELETE)
- .status(AuditStatus.SUCCESS)
- .actionId(INDEX_DELETE_RECORD_ACTION_ID)
- .message(INDEX_DELETE_RECORDS_SUCCESS)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexDeleteRecordFailEvent(List<String> resources) {
- return builder()
- .action(AuditAction.DELETE)
- .status(AuditStatus.FAILURE)
- .actionId(INDEX_DELETE_RECORD_ACTION_ID)
- .message(INDEX_DELETE_RECORDS_FAILURE)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexPurgeRecordSuccessEvent(List<String> resources) {
- return builder()
- .action(AuditAction.DELETE)
- .status(AuditStatus.SUCCESS)
- .actionId(INDEX_PURGE_RECORD_ACTION_ID)
- .message(INDEX_DELETE_RECORDS_SUCCESS)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexPurgeRecordFailEvent(List<String> resources) {
- return builder()
- .action(AuditAction.DELETE)
- .status(AuditStatus.FAILURE)
- .actionId(INDEX_PURGE_RECORD_ACTION_ID)
- .message(INDEX_DELETE_RECORDS_FAILURE)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getReindexEvent(List<String> resources) {
- return builder()
- .action(AuditAction.CREATE)
- .status(AuditStatus.SUCCESS)
- .actionId(REINDEX_KIND_ACTION_ID)
- .message(REINDEX_KIND_OPERATION)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getCopyIndexEvent(List<String> resources) {
- return builder()
- .action(AuditAction.CREATE)
- .status(AuditStatus.SUCCESS)
- .actionId(COPY_INDEX_ACTION_ID)
- .message(COPY_INDEX_OPERATION)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getTaskStatusEvent(List<String> resources) {
- return builder()
- .action(AuditAction.READ)
- .status(AuditStatus.SUCCESS)
- .actionId(GET_TASK_STATUS_ACTION_ID)
- .message(GET_TASK_STATUS_OPERATION)
- .resources(resources)
- .user(this.user)
- .build();
- }
-
- public AuditPayload getIndexCleanUpJobRunEvent(List<String> resources) {
- return builder()
- .action(AuditAction.JOB_RUN)
- .status(AuditStatus.SUCCESS)
- .user(this.user)
- .actionId(RUN_JOB_ACTION_ID)
- .message(RUN_JOB_MESSAGE_SUCCESS)
- .resources(resources)
- .build();
- }
-
- public AuditPayload getIndexMappingUpdateEvent(List<String> resources, boolean isSuccess) {
- if (isSuccess) {
- return builder()
- .action(AuditAction.UPDATE)
- .status(AuditStatus.SUCCESS)
- .actionId(INDEX_MAPPING_UPDATE_ACTION_ID)
- .message(INDEX_MAPPING_UPDATE_SUCCESS)
- .resources(resources)
- .user(this.user)
- .build();
- } else {
- return builder()
- .action(AuditAction.UPDATE)
- .status(AuditStatus.FAILURE)
- .actionId(INDEX_MAPPING_UPDATE_ACTION_ID)
- .message(INDEX_MAPPING_UPDATE_FAILURE)
- .resources(resources)
- .user(this.user)
- .build();
- }
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
deleted file mode 100644
index 7bd9e97c3568b1d6f93c7c0d624b797c701d85e4..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.logging;
-
-import org.opengroup.osdu.core.logging.payload.AuditPayload;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import java.util.List;
-
-@Component
-@RequestScope
-public class AuditLogger {
-
- @Inject
- private JaxRsDpsLog logger;
- @Inject
- private IHeadersInfo headers;
-
- private AuditEvents events = null;
-
- private AuditEvents getAuditEvents() {
- if (this.events == null) {
- this.events = new AuditEvents(this.headers.getUser());
- }
- return this.events;
- }
-
- public void indexCreateRecordSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexCreateRecordSuccessEvent(resources));
- }
-
- public void indexCreateRecordFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexCreateRecordFailEvent(resources));
- }
-
- public void indexUpdateRecordSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexUpdateRecordSuccessEvent(resources));
- }
-
- public void indexUpdateRecordFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexUpdateRecordFailEvent(resources));
- }
-
- public void indexDeleteRecordSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexDeleteRecordSuccessEvent(resources));
- }
-
- public void indexDeleteRecordFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexDeleteRecordFailEvent(resources));
- }
-
- public void indexPurgeRecordSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexPurgeRecordSuccessEvent(resources));
- }
-
- public void indexPurgeRecordFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexPurgeRecordFailEvent(resources));
- }
-
- public void getReindex(List<String> resources) {
- this.writeLog(this.getAuditEvents().getReindexEvent(resources));
- }
-
- public void copyIndex(List<String> resources) {
- this.writeLog(this.getAuditEvents().getCopyIndexEvent(resources));
- }
-
- public void getTaskStatus(List<String> resources) {
- this.writeLog(this.getAuditEvents().getTaskStatusEvent(resources));
- }
-
- public void getIndexCleanUpJobRun(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexCleanUpJobRunEvent(resources));
- }
-
- public void indexMappingUpdateSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,true));
- }
- public void indexMappingUpdateFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,false));
- }
-
- private void writeLog(AuditPayload log) {
- this.logger.audit(log);
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/middleware/AuthorizationFilter.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/middleware/AuthorizationFilter.java
index fbdbd0669fd30b1d5b0e7efbba90254919cf87a3..2cdf3fbd6dc6b00705093bf0091bb44d2f2203ac 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/middleware/AuthorizationFilter.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/middleware/AuthorizationFilter.java
@@ -2,9 +2,9 @@ package org.opengroup.osdu.indexer.middleware;
import com.google.common.base.Strings;
import lombok.extern.java.Log;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.model.AuthorizationResponse;
-import org.opengroup.osdu.is.core.provider.interfaces.auth.AuthorizationService;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AuthorizationResponse;
+import org.opengroup.osdu.core.common.spi.coreis.AuthorizationService;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ElasticType.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ElasticType.java
deleted file mode 100644
index 49c9eee08bf7ca62b6de14068ed1aa15a54b54e5..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ElasticType.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import com.google.common.base.Strings;
-
-public enum ElasticType {
-
- KEYWORD("keyword"),
-
- TEXT("text"),
-
- DATE("date"),
-
- NESTED("nested"),
-
- OBJECT("object"),
-
- GEO_POINT("geo_point"),
-
- GEO_SHAPE("geo_shape"),
-
- INTEGER("integer"),
-
- LONG("long"),
-
- FLOAT("float"),
-
- DOUBLE("double"),
-
- BOOLEAN("boolean"),
-
- UNDEFINED("undefined");
-
- private final String value;
-
- ElasticType(String value) {
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-
- public static ElasticType forValue(String value) {
-
- if (Strings.isNullOrEmpty(value)) return ElasticType.UNDEFINED;
-
- for (ElasticType type : values()) {
- if (type.getValue().equalsIgnoreCase(value)) {
- return type;
- }
- }
- return ElasticType.UNDEFINED;
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexProgress.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexProgress.java
deleted file mode 100644
index 7a43e3a3fe13254c98186d60b7af3c2e5fc19f95..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexProgress.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.Builder;
-import lombok.Data;
-
-import java.util.Stack;
-
-@Data
-@Builder
-public class IndexProgress {
- private int statusCode;
- private Stack<String> trace;
- private String lastUpdateTime;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexSchema.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexSchema.java
deleted file mode 100644
index 41ebf5dde825dc6e5998edae4662aa33b45fbdb0..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexSchema.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.Builder;
-import lombok.Data;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-@Data
-@Builder
-public class IndexSchema {
-
- private String kind;
- private String type;
- private Map<String, String> dataSchema;
- private Map<String, Object> metaSchema;
-
- public ArrayList<String> getSchemaKeysByValue(String value) {
- Set<String> keys = new HashSet<>();
- for (Map.Entry<String, String> entry : this.getDataSchema().entrySet()) {
- if (value.equalsIgnoreCase(entry.getValue())) {
- keys.add(entry.getKey());
- }
- }
- return new ArrayList<>(keys);
- }
-
- public boolean isDataSchemaMissing() {
- return dataSchema == null || dataSchema.isEmpty();
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Legal.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Legal.java
deleted file mode 100644
index e58b8992077bcf75cfe60f847f0b57735558feaa..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Legal.java
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.Data;
-import org.hibernate.validator.constraints.NotEmpty;
-
-import javax.validation.constraints.NotNull;
-
-@Data
-public class Legal {
-
- @NotNull
- @NotEmpty
- private String[] legaltags;
-
- @NotNull
- @NotEmpty
- private String[] otherRelevantDataCountries;
-
- private String status;
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/MultiFieldIndexRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/MultiFieldIndexRequest.java
deleted file mode 100644
index 907cb4d80f900250adcb0bdfced64c84b6af9644..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/MultiFieldIndexRequest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.hibernate.validator.constraints.NotBlank;
-import org.hibernate.validator.constraints.NotEmpty;
-
-import javax.validation.constraints.NotNull;
-import javax.validation.constraints.Size;
-import java.util.Set;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class MultiFieldIndexRequest {
- @NotNull
- @NotEmpty
- private Set<String> indices;
- @NotBlank
- @Size(min=1, max=10)
- @Builder.Default
- private String operator = "";
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/OperationType.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/OperationType.java
deleted file mode 100644
index 949e9e7272fc6da2c38feed34dcac47c88974307..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/OperationType.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-public enum OperationType {
- /**
- * A post operation
- */
- create("create"),
-
- /**
- * A delete operation
- */
- delete("delete"),
-
- /**
- * A purge operation
- */
- purge("purge"),
-
- /**
- * A patch operation
- */
- update("update"),
-
- /*
- * create schema operation
- * */
- create_schema("create_schema"),
-
- /*
- * purge schema operation
- * */
- purge_schema("purge_schema");
-
- private final String value;
-
- OperationType(String value) {
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIds.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIds.java
deleted file mode 100644
index 2ae54437a0f9c2c0857846853784f39b671df312..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIds.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RecordIds {
- private List<String> records;
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIndexerPayload.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIndexerPayload.java
deleted file mode 100644
index c6fcf9d0b7a8e41a96414275b4334dd1c4cda347..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordIndexerPayload.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RecordIndexerPayload {
-
- private List<IndexSchema> schemas;
- private List<Record> records;
-
- @Data
- public static class Record {
- private String id;
- private String kind;
- private String namespace;
- private String type;
- private OperationType operationType;
- private long version;
- private StorageAcl acl;
- private IndexProgress indexProgress;
- private Legal legal;
- private RecordAncestry ancestry;
- private Map<String, Object> data;
- @JsonIgnore
- private boolean schemaMissing = false;
- @JsonIgnore
- private boolean mappingMismatch = false;
-
- public boolean skippedDataIndexing() {
- return schemaMissing || mappingMismatch;
- }
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordQueryResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordQueryResponse.java
deleted file mode 100644
index 0d011379260db446c3f922887434338b849ba6b2..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordQueryResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class RecordQueryResponse {
- private String cursor;
- private List<String> results;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordReindexRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordReindexRequest.java
deleted file mode 100644
index fc968b66477bb322d72dc26dd75625b23dd389e8..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordReindexRequest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.hibernate.validator.constraints.NotBlank;
-import org.opengroup.osdu.indexer.SwaggerDoc;
-import org.opengroup.osdu.is.core.validation.ValidKind;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class RecordReindexRequest {
-
- @NotBlank(message = SwaggerDoc.KIND_VALIDATION_CAN_NOT_BE_NULL_OR_EMPTY)
- @ValidKind
- private String kind;
-
- private String cursor;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Records.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Records.java
deleted file mode 100644
index 8874d96b46036bd114397b9aba7fe6f498d0f975..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Records.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.*;
-
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class Records {
-
- @Singular
- private List<Entity> records;
- private List<String> notFound;
- @Singular
- private List<ConversionStatus> conversionStatuses;
-
- @Data
- @Builder
- @NoArgsConstructor
- @AllArgsConstructor
- @JsonInclude(JsonInclude.Include.NON_NULL)
- @JsonIgnoreProperties(ignoreUnknown = true)
- public static class Entity {
- private String id;
- private long version;
- private String kind;
- private StorageAcl acl;
- private Legal legal;
- private RecordAncestry ancestry;
- private Map<String, Object> data;
- private List<Object> meta;
- }
-
- @Data
- @Builder
- public static class Type {
- private String type;
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageAcl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageAcl.java
deleted file mode 100644
index bfcd1c30b5dd3f009a085e53988367d332ee58d3..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageAcl.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.Data;
-import org.hibernate.validator.constraints.NotEmpty;
-
-import javax.validation.constraints.NotNull;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-@Data
-public class StorageAcl {
-
- @NotNull
- @NotEmpty
- private String[] viewers;
-
- @NotNull
- @NotEmpty
- private String[] owners;
-
- public static String[] flattenAcl(StorageAcl acl) {
- Set<String> xAcl = new HashSet<>();
- if (acl.getOwners() != null && acl.getOwners().length > 0) xAcl.addAll(Arrays.asList(acl.getOwners()));
- if (acl.getViewers() != null && acl.getViewers().length > 0) xAcl.addAll(Arrays.asList(acl.getViewers()));
- return xAcl.toArray(new String[xAcl.size()]);
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageType.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageType.java
deleted file mode 100644
index 433bc8f28916a4508bc9cbf516330adfe34b5478..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/StorageType.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.model;
-
-public enum StorageType {
-
- LINK("link"),
-
- LINK_ARRAY("[]link"),
-
- BOOLEAN("boolean"),
-
- STRING("string"),
-
- INT("int"),
-
- FLOAT("float"),
-
- DOUBLE("double"),
-
- DOUBLE_ARRAY("[]double"),
-
- LONG("long"),
-
- DATETIME("datetime"),
-
- GEO_POINT("core:dl:geopoint:1.0.0"),
-
- GEO_SHAPE("core:dl:geoshape:1.0.0");
-
- private final String value;
-
- StorageType(String value) {
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/IPublisher.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/IPublisher.java
deleted file mode 100644
index c5eb831196aa301400cc3d5c04cc024165e528cb..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/IPublisher.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.opengroup.osdu.indexer.provider.interfaces;
-
-import org.opengroup.osdu.indexer.util.JobStatus;
-import org.opengroup.osdu.core.api.DpsHeaders;
-
-public interface IPublisher {
-
- public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java
deleted file mode 100644
index c8b499feb21d2e9caad80fa246339e6d787fd646..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/provider/interfaces/ISchemaCache.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.opengroup.osdu.indexer.provider.interfaces;
-
-import org.opengroup.osdu.core.cache.ICache;
-
-public interface ISchemaCache <String,V> extends ICache<String, V> {
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 370194f3ee393464186e1937aea46bbc2200e6b5..f737ad6beda38488059342def67382cff48315f6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -19,16 +19,14 @@ import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;
-
import org.apache.http.HttpStatus;
-import org.opengroup.osdu.indexer.model.ElasticType;
-import org.opengroup.osdu.indexer.model.IndexSchema;
-import org.opengroup.osdu.indexer.model.IndexingStatus;
-import org.opengroup.osdu.indexer.util.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.core.Constants;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
-
-import org.opengroup.osdu.is.core.util.Constants;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java
index 268f8196b7b8f8b2acafd4bfafd1585c55d8563f..7478b7b6c0bd82ff8359f5559095379e5c099416 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/CronServiceImpl.java
@@ -18,11 +18,11 @@ package org.opengroup.osdu.indexer.service;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.IndexInfo;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IRequestInfo;
-import org.opengroup.osdu.is.core.service.IndicesService;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.IndexInfo;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
import org.opengroup.osdu.is.core.util.ElasticClientHandler;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
index e93c26135295ae7116723beb3733c66bc921fa92..99329dc643ba84c4a17d338b8c3fd84cad4be94a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceImpl.java
@@ -15,14 +15,14 @@
package org.opengroup.osdu.indexer.service;
import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.multitenancy.TenantInfo;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.ClusterSettings;
-import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache;
-import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.ElasticRepository;
+import org.opengroup.osdu.core.common.spi.coreis.IElasticCredentialsCache;
+import org.opengroup.osdu.core.common.service.coreis.TenantInfoService;
import org.opengroup.osdu.is.core.service.ElasticSettingService;
-import org.opengroup.osdu.is.core.service.TenantInfoService;
-import org.opengroup.osdu.is.core.util.AppException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
index 0b680e54268336f58e9058ec907c38b4672672a1..e032c66b3c59aa08c0fa2c104dab1871b1e70e93 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
@@ -15,7 +15,7 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.internal.LinkedTreeMap;
-import org.opengroup.osdu.is.core.util.Constants;
+import org.opengroup.osdu.core.common.model.core.Constants;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index b86ebc2ed2a6090f11cc2598874f7ed8baf8d024..edba8e24f8d4c306d566191b1df8ce188c74d955 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -1,6 +1,6 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.indexer.model.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import java.util.Map;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyService.java
index 4bf50e13b6b4c122033d8f6754e9c08129cba26d..465c28aed3f5262f8719eb1f7a287a8f803a8a7d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyService.java
@@ -14,7 +14,8 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.is.core.util.AppException;
+
+import org.opengroup.osdu.core.common.model.coreis.AppException;
import java.io.IOException;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java
index b8df1edacb279d0667dc8b8c6368f7f0ded6d1d1..966c796a755b9543dbbb26b30843ae48b03b22bb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImpl.java
@@ -25,14 +25,15 @@ import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.multitenancy.TenantInfo;
-import org.opengroup.osdu.is.core.model.ClusterSettings;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.*;
+import org.opengroup.osdu.core.common.model.indexer.AuditLogger;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
import org.opengroup.osdu.is.core.service.ElasticSettingService;
-import org.opengroup.osdu.is.core.service.IndicesService;
-import org.opengroup.osdu.is.core.util.*;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index 092d66524046811ac22158df660eced309d79999..02257801e8d23649c55ed04c38a317be5a7761e9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -14,9 +14,9 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.indexer.model.IndexSchema;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
import java.io.IOException;
import java.util.Map;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index ba6d69c7cd8e5dac55b145352e177e68b868c077..12a84303f7586f838db9a0751c5dc5c39b09579b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -18,19 +18,19 @@ import com.google.common.base.Strings;
import com.google.gson.Gson;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.indexer.model.IndexSchema;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.indexer.model.Schema;
-import org.opengroup.osdu.indexer.model.StorageType;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.opengroup.osdu.indexer.util.TypeMapper;
-import org.opengroup.osdu.is.core.httpclient.RequestStatus;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.RecordMetaAttribute;
-import org.opengroup.osdu.is.core.service.IndicesService;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
import org.opengroup.osdu.is.core.util.ElasticClientHandler;
-import org.opengroup.osdu.is.core.util.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.service.coreis.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.Schema;
+import org.opengroup.osdu.core.common.model.indexer.StorageType;
+import org.opengroup.osdu.core.common.spi.indexer.ISchemaCache;
+import org.opengroup.osdu.indexer.util.TypeMapper;
+import org.opengroup.osdu.core.common.model.coreis.RequestStatus;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
import org.apache.http.HttpStatus;
import org.springframework.stereotype.Service;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
index e5d1a64c00b0b48d889d8f9867447f1f6b230c73..ad1b3d4f2a3dcaf52228412c6117ac5d28682563 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
@@ -15,8 +15,8 @@
package org.opengroup.osdu.indexer.service;
import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.indexer.model.IndexSchema;
-import org.opengroup.osdu.is.core.service.MappingService;
+import org.opengroup.osdu.core.common.service.coreis.MappingService;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import java.io.IOException;
import java.util.Map;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index ba2aecad45ddd34756881e72511c197532a1067e..65f5cca91a44bf765272f092af66b195f7f09e87 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -37,15 +37,15 @@ import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import com.google.gson.Gson;
-import org.opengroup.osdu.indexer.model.IndexSchema;
-import org.opengroup.osdu.indexer.model.Records;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.model.core.Constants;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.service.coreis.Preconditions;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.is.core.service.MappingServiceImpl;
-import org.opengroup.osdu.is.core.util.AppException;
-import org.opengroup.osdu.is.core.util.Constants;
import org.opengroup.osdu.is.core.util.ElasticClientHandler;
-import org.opengroup.osdu.is.core.util.Preconditions;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java
index ccffa37480b0e3500ec733cce24b459ea362776c..88bba90c9e1e72eb3b22f37c12d54a23ba2919c1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java
@@ -16,9 +16,9 @@ package org.opengroup.osdu.indexer.service;
import java.util.List;
-import org.opengroup.osdu.indexer.util.JobStatus;
-import org.opengroup.osdu.indexer.util.RecordInfo;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
public interface IndexerService {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index c24aca4913c3066e513524c9fbca9509378fd198..953aaa9e987b94149805fd169c9d398b2ac14cb0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -30,23 +30,20 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.model.*;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.core.common.model.core.Constants;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.indexer.IPublisher;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.indexer.util.JobStatus;
-import org.opengroup.osdu.indexer.util.RecordInfo;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.httpclient.RequestStatus;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
-import org.opengroup.osdu.is.core.model.RecordMetaAttribute;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IRequestInfo;
-import org.opengroup.osdu.is.core.service.IndicesService;
-import org.opengroup.osdu.is.core.util.AppException;
-import org.opengroup.osdu.is.core.util.Constants;
+import org.opengroup.osdu.core.common.model.coreis.RequestStatus;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
import org.opengroup.osdu.is.core.util.ElasticClientHandler;
-import org.opengroup.osdu.is.core.util.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.service.coreis.ElasticIndexNameResolver;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.beanutils.NestedNullException;
import org.springframework.stereotype.Service;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 1e41efdaaacb79d8e068b475f6811e4a812b5875..40f22aaa26f931f6bde8aa8d0d28f33a61cbc16e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -15,7 +15,7 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
public interface ReindexService {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index b0b5c94e20856b20383cbdc0f644dcb0bd9a8643..f887b994e1fee5ee0a38d0de0febd1e7f4814231 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -18,16 +18,16 @@ import com.google.common.base.Strings;
import com.google.gson.Gson;
import lombok.extern.java.Log;
import org.apache.http.HttpStatus;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.indexer.model.RecordQueryResponse;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.indexer.util.RecordInfo;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IRequestInfo;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
index 812b39bc53c3575c6a9dfe374ec321b52ce11c1c..6c7de1f5e9f425802e18e6730cc128789f8dc698 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
@@ -14,10 +14,10 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.indexer.model.RecordQueryResponse;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
-import org.opengroup.osdu.indexer.model.Records;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 485f3e98754644262296abb4790ef9bf32381756..21e383aa7be974a5a7a7d74e29b399b4ec0a03af 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -20,16 +20,15 @@ import com.google.common.collect.Lists;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import lombok.extern.java.Log;
-import org.opengroup.osdu.indexer.model.*;
-import org.opengroup.osdu.indexer.util.JobStatus;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.httpclient.HttpResponse;
-import org.opengroup.osdu.is.core.httpclient.RequestStatus;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.RecordMetaAttribute;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IRequestInfo;
-import org.opengroup.osdu.is.core.service.UrlFetchService;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.HttpResponse;
+import org.opengroup.osdu.core.common.model.coreis.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.service.coreis.UrlFetchService;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
import org.apache.http.HttpStatus;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -44,8 +43,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.opengroup.osdu.is.core.model.SlbHeaders.SLB_FRAME_OF_REFERENCE;
-import static org.opengroup.osdu.is.core.util.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
+import static org.opengroup.osdu.core.common.model.coreis.SlbHeaders.SLB_FRAME_OF_REFERENCE;
+import static org.opengroup.osdu.core.common.model.core.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
@Log
@Component
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/TenantInfoServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/TenantInfoServiceImpl.java
index 0ec16d2fe4839b1f429ca60cc12a62095f3427c2..fd71a5597050a52332887588388bc3fba47bad56 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/TenantInfoServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/TenantInfoServiceImpl.java
@@ -15,11 +15,12 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.multitenancy.ITenantFactory;
-import org.opengroup.osdu.core.multitenancy.TenantInfo;
-import org.opengroup.osdu.is.core.service.TenantInfoService;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.TenantInfoService;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -29,9 +30,11 @@ public class TenantInfoServiceImpl implements TenantInfoService {
@Inject
private ITenantFactory tenantFactory;
@Inject
+ @Qualifier("dpsHeaderFactorySearch")
private DpsHeaders headersInfo;
@Inject
+ @Qualifier("TenantInfoFactorySearch")
private TenantInfo tenantInfo;
public TenantInfo getTenantInfo() {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
index 0b844e122d212bde1e3841fcdbb0af6529a39749..fd724bc3318f5251d9638e5b5770ebb270473b3a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
@@ -17,11 +17,11 @@ package org.opengroup.osdu.indexer.util;
import com.google.api.client.http.HttpMethods;
import com.google.gson.Gson;
import lombok.extern.java.Log;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.httpclient.HttpResponse;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.CloudTaskRequest;
-import org.opengroup.osdu.is.core.service.UrlFetchService;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.CloudTaskRequest;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.coreis.HttpResponse;
+import org.opengroup.osdu.core.common.service.coreis.UrlFetchService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
@@ -29,8 +29,8 @@ import org.springframework.web.context.annotation.RequestScope;
import java.net.URISyntaxException;
import javax.inject.Inject;
-import static org.opengroup.osdu.is.core.util.Constants.REINDEX_RELATIVE_URL;
-import static org.opengroup.osdu.is.core.util.Constants.WORKER_RELATIVE_URL;
+import static org.opengroup.osdu.core.common.model.core.Constants.REINDEX_RELATIVE_URL;
+import static org.opengroup.osdu.core.common.model.core.Constants.WORKER_RELATIVE_URL;
@Log
@Component
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/JobStatus.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/JobStatus.java
deleted file mode 100644
index a246228e8ff78b0a9afca0c5799344a55371fe46..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/JobStatus.java
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.common.base.Strings;
-import lombok.extern.java.Log;
-import lombok.Data;
-import org.opengroup.osdu.indexer.model.IndexProgress;
-import org.opengroup.osdu.indexer.model.IndexingStatus;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.indexer.model.RecordStatus;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import java.time.Instant;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Log
-@Data
-@Component
-@RequestScope
-public class JobStatus {
-
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
-
- private List<RecordStatus> statusesList = new ArrayList<>();
-
- private List<String> debugInfos = new ArrayList<>();
-
- public void initialize(List<RecordInfo> recordInfos) {
-
- if (recordInfos == null || recordInfos.isEmpty()) return;
-
- List<RecordStatus> statuses = recordInfos.stream().map(msg -> RecordStatus.builder()
- .id(msg.getId())
- .kind(msg.getKind())
- .operationType(msg.getOp())
- .status(IndexingStatus.PROCESSING)
- .indexProgress(IndexProgress.builder().trace(new Stack<>()).lastUpdateTime(Instant.now().toString()).build())
- .build()).collect(Collectors.toList());
-
- this.statusesList.addAll(statuses);
- }
-
- public void addOrUpdateRecordStatus(Collection<String> ids, IndexingStatus status, int statusCode, String message, String debugInfo) {
-
- this.debugInfos.add(debugInfo);
- addOrUpdateRecordStatus(ids, status, statusCode, message);
- }
-
- public void addOrUpdateRecordStatus(String id, IndexingStatus status, int statusCode, String message, String debugInfo) {
-
- this.debugInfos.add(debugInfo);
- addOrUpdateRecordStatus(id, status, statusCode, message);
- }
-
- public void addOrUpdateRecordStatus(Collection<String> ids, IndexingStatus status, int statusCode, String message) {
-
- if (ids == null || ids.isEmpty()) return;
- ids.forEach(id -> addOrUpdateRecordStatus(id, status, statusCode, message));
- }
-
- public void addOrUpdateRecordStatus(String id, IndexingStatus status, int statusCode, String message) {
- Optional<RecordStatus> queryResult = this.statusesList.stream().filter(s -> s.getId().equalsIgnoreCase(id)).findFirst();
- if (queryResult.isPresent()) {
- RecordStatus s = queryResult.get();
- IndexProgress indexProgress = s.getIndexProgress();
- indexProgress.setStatusCode(statusCode);
- indexProgress.setLastUpdateTime(Instant.now().toString());
- if (!Strings.isNullOrEmpty(message)) {
- indexProgress.getTrace().add(message);
- }
- if (status.isWorseThan(s.getStatus())) {
- s.setStatus(status);
- }
- s.setIndexProgress(indexProgress);
- } else {
- IndexProgress indexProgress = IndexProgress.builder()
- .trace(new Stack<>())
- .lastUpdateTime(Instant.now().toString()).build();
- indexProgress.getTrace().add(message);
- this.statusesList.add(RecordStatus.builder().id(id).status(status).indexProgress(indexProgress).build());
- }
- }
-
- public List<String> getIdsByIndexingStatus(IndexingStatus indexingStatus) {
-
- return this.statusesList.stream().filter(s -> s.getStatus() == indexingStatus).map(RecordStatus::getId)
- .collect(Collectors.toList());
- }
-
- public String getRecordKindById(String id) {
- Optional<RecordStatus> optionalRecordStatus = this.statusesList.stream().filter(s -> s.getId()
- .equalsIgnoreCase(id)).findFirst();
- RecordStatus status = optionalRecordStatus.orElse(null);
- return status != null ? status.getKind() : null;
- }
-
- public RecordStatus getJobStatusByRecordId(String id) {
- Optional<RecordStatus> optionalRecordStatus = this.statusesList.stream().filter(s -> s.getId()
- .equalsIgnoreCase(id)).findFirst();
- return optionalRecordStatus.orElse(null);
- }
-
- public List<RecordStatus> getRecordStatuses(IndexingStatus indexingStatus, OperationType operationType) {
- return this.statusesList.stream().filter(
- s -> s.getStatus() == indexingStatus && s.getOperationType().equalsIgnoreCase(operationType.getValue())).collect(Collectors.toList());
- }
-
- /*
- * mark all the records as FAIL if for some reason they were not processed
- * */
- public void finalizeRecordStatus(String errorMessage) {
- statusesList.stream().filter(recordStatus -> recordStatus.getStatus() == IndexingStatus.PROCESSING).forEach
- (recordStatus -> {
- recordStatus.setStatus(IndexingStatus.FAIL);
- recordStatus.getIndexProgress().getTrace().add(errorMessage);
- });
-
- // dump all debug-info
- this.jaxRsDpsLog.warning(this.debugInfos);
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/RecordInfo.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/RecordInfo.java
deleted file mode 100644
index 0895ea614c5fb406f9ab5e5e989317e19a1d8e5c..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/RecordInfo.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.is.core.util.AppException;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RecordInfo {
-
- private static final long serialVersionUID = 1L;
-
- private String id;
- private String kind;
- private String op;
-
- public static Map<String, Map<String, OperationType>> getUpsertRecordIds(List<RecordInfo> msgs) throws AppException {
-
- Map<String, Map<String, OperationType>> kindRecordOpMap = new HashMap<>();
-
- try {
- for (RecordInfo msg : msgs) {
- OperationType op = OperationType.valueOf(msg.getOp());
- if (op == OperationType.create || op == OperationType.update) {
- Map<String, OperationType> idOperationMap = kindRecordOpMap.containsKey(msg.getKind()) ? kindRecordOpMap.get(msg.getKind()) : new HashMap<>();
- idOperationMap.put(msg.getId(), OperationType.valueOf(msg.getOp()));
- kindRecordOpMap.put(msg.getKind(), idOperationMap);
- }
- }
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing upsert records in request payload.", e);
- }
- return kindRecordOpMap;
- }
-
- public static Map<String, List<String>> getDeleteRecordIds(List<RecordInfo> msgs) {
-
- Map<String, List<String>> deleteRecordMap = new HashMap<>();
-
- try {
- for (RecordInfo msg : msgs) {
- OperationType op = OperationType.valueOf(msg.getOp());
- if (op == OperationType.purge || op == OperationType.delete) {
- String kind = msg.getKind();
- if (!deleteRecordMap.containsKey(kind)) {
- deleteRecordMap.put(kind, new ArrayList<>());
- }
- deleteRecordMap.get(kind).add(msg.getId());
- }
- }
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing delete records in request payload.", e);
- }
- return deleteRecordMap;
- }
-
- public static Map<String, OperationType> getSchemaMsgs(List<RecordInfo> msgs) {
-
- Map<String, OperationType> schemaOperations = new HashMap<>();
-
- try {
- for (RecordInfo msg : msgs) {
- OperationType op = OperationType.valueOf(msg.getOp());
- if (op == OperationType.create_schema || op == OperationType.purge_schema) {
- schemaOperations.put(msg.getKind(), op);
- }
- }
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Request parsing error", "Error parsing schema updates in request payload.", e);
- }
- return schemaOperations;
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index f69504c2f94870db63245add199fd853bb2114c7..4241dbf9c90150cb930ae74d559319f14a828b49 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -14,12 +14,12 @@
package org.opengroup.osdu.indexer.util;
-import org.opengroup.osdu.indexer.model.ElasticType;
-import org.opengroup.osdu.indexer.model.Records;
-import org.opengroup.osdu.indexer.model.StorageType;
-import org.opengroup.osdu.is.core.model.AclRole;
-import org.opengroup.osdu.is.core.model.RecordMetaAttribute;
-import org.opengroup.osdu.is.core.util.Constants;
+import org.opengroup.osdu.core.common.model.core.Constants;
+import org.opengroup.osdu.core.common.model.coreis.AclRole;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.model.indexer.StorageType;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
import java.util.HashMap;
import java.util.Map;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
index 573be342c220f5e121c87471295dd9f9ac6cf851..d8106c5635d443202dbfc1d8633fc35f544331f1 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
@@ -22,15 +22,15 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.Config;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.DeploymentEnvironment;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
-import org.opengroup.osdu.is.core.util.AppException;
-import org.opengroup.osdu.is.core.util.Config;
-import org.opengroup.osdu.is.core.util.HeadersUtil;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
+import org.opengroup.osdu.core.common.service.coreis.Config;
+import org.opengroup.osdu.core.common.service.coreis.HeadersUtil;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.http.HttpStatus;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index 5b8b59ee3c215f01049674b2b318a64183a1162f..bc33aaf4ccf858631f6af3b921c6bd5d1187704a 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -19,11 +19,11 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.model.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.AuditLogger;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.is.core.util.AppException;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit4.SpringRunner;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlementsTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlementsTest.java
index 1434113f94c1b09f29e5abdfadc17cb11a8ad642..73533139e713ab0b5059931baa375e82345d7462 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlementsTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/auth/AuthorizationServiceEntitlementsTest.java
@@ -19,16 +19,16 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.api.entitlements.EntitlementsException;
-import org.opengroup.osdu.core.api.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.api.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.api.entitlements.models.GroupInfo;
-import org.opengroup.osdu.core.api.entitlements.models.Groups;
-import org.opengroup.osdu.core.httpclient.HttpResponse;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.model.AuthorizationResponse;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.core.entitlements.EntitlementsException;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+import org.opengroup.osdu.core.common.model.core.entitlements.GroupInfo;
+import org.opengroup.osdu.core.common.model.core.entitlements.Groups;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.core.HttpResponse;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.coreis.AuthorizationResponse;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
index 3d9fee13e5c2e0585509a7963710943c7372d13a..9a5b09a3a57da2f9be4ecfda697b5efe739f225b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
@@ -17,8 +17,9 @@ package org.opengroup.osdu.indexer.logging;
import com.google.common.collect.Lists;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.opengroup.osdu.core.logging.payload.AuditAction;
-import org.opengroup.osdu.core.logging.payload.AuditStatus;
+import org.opengroup.osdu.core.common.model.core.AuditAction;
+import org.opengroup.osdu.core.common.model.core.AuditStatus;
+import org.opengroup.osdu.core.common.model.indexer.AuditEvents;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.Map;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
index 1f67755bf9749bf942ae275af56cbc8b9a3b75da..a8cfcbffb8a35c9cde44738bcb9d1451c55cb92a 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
@@ -21,9 +21,10 @@ import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.logging.payload.AuditPayload;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
-import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
+import org.opengroup.osdu.core.common.model.core.AuditPayload;
+import org.opengroup.osdu.core.common.model.indexer.AuditLogger;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.Map;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/JaxRsDpsLogTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/JaxRsDpsLogTest.java
index e4a44ae0258fe6ea761853dd913c5c2289f892fd..bd5624bd54223d8fb2086c324226d2edcfac4d61 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/JaxRsDpsLogTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/JaxRsDpsLogTest.java
@@ -20,11 +20,12 @@ import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.logging.DpsLog;
-import org.opengroup.osdu.core.logging.payload.AuditPayload;
-import org.opengroup.osdu.core.logging.payload.Request;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.core.AuditPayload;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.DpsLog;
+import org.opengroup.osdu.core.common.model.core.Request;
+import org.opengroup.osdu.core.common.model.indexer.ServiceLogId;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/ElasticTypeTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/ElasticTypeTest.java
index 271b0b6717e237b657e59f3ace3d15fadd7647be..99696cef1c6597d43d29a93771aae8e7c87ccc3a 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/ElasticTypeTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/ElasticTypeTest.java
@@ -15,9 +15,9 @@
package org.opengroup.osdu.indexer.model;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/IndexingStatusTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/IndexingStatusTest.java
index 61ccf7a0eadcc771445826968ddb80bcde5a667b..a9febb4c1dd141445d1b43a3ec1c790df5e1c25c 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/IndexingStatusTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/IndexingStatusTest.java
@@ -15,9 +15,9 @@
package org.opengroup.osdu.indexer.model;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/RecordChagedMessagesTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/RecordChagedMessagesTest.java
index 1cc0a345643d20475888f2439b60ac2d9cbfd679..de936e7aac444684806a6ee7b5f658650480793d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/RecordChagedMessagesTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/RecordChagedMessagesTest.java
@@ -18,8 +18,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.is.core.model.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.HashMap;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/StorageTypeTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/StorageTypeTest.java
index d41338ffac12e05c2cde94f557c3632e60faacc7..c85a6ce8fe5ca8ff7f320649d7f633526877de25 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/StorageTypeTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/StorageTypeTest.java
@@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.model;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
index 86d2bb96b3cac4d61a597b13be3606f7c21f75b7..8c369cb72b43c5f3b07467bdddf9d07bb929408b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
@@ -19,10 +19,10 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.indexer.util.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.HashMap;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/GeometryConversionServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/GeometryConversionServiceTest.java
index 52d1c11948ee5ebea304199ab8a08d566e42bde0..e5cf8889a53cc411db4bddddf39e937a77963081 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/GeometryConversionServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/GeometryConversionServiceTest.java
@@ -18,7 +18,7 @@ import com.google.gson.internal.LinkedTreeMap;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
-import org.opengroup.osdu.is.core.util.Constants;
+import org.opengroup.osdu.core.common.model.core.Constants;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
index bb572412db582bace8b91ea9f970dc2c5dfece92..d2d19299c9dd3b68cd39e28a44ce28ef0e771f38 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
@@ -20,11 +20,13 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.indexer.model.IndexingStatus;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.indexer.model.RecordStatus;
-import org.opengroup.osdu.indexer.model.Records;
-import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.springframework.test.context.junit4.SpringRunner;
import java.lang.reflect.Type;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/RecordInfoTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/RecordInfoTest.java
index 1194c6a7bfd948c67385284675a1733c90f004a1..f06fbd6e18091e021bc44ca9391b04049047bf9d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/RecordInfoTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/RecordInfoTest.java
@@ -18,11 +18,11 @@ import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.opengroup.osdu.indexer.model.OperationType;
-import org.opengroup.osdu.is.core.util.AppException;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.springframework.http.HttpStatus;
import org.springframework.test.context.junit4.SpringRunner;
diff --git a/pom.xml b/pom.xml
index 0a9ad9be3c7e0a0c74773d97b3143054d90c9284..e52bd5532ba7c8ce8ab46e141e35ef336a8ca068 100644
--- a/pom.xml
+++ b/pom.xml
@@ -14,7 +14,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>1.0.0</version>
+ <version>1.0.2</version>
<description>Indexer Service</description>
<properties>
@@ -48,12 +48,6 @@
<repository>
<id>os-core</id>
<url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
</repository>
</distributionManagement>
@@ -64,11 +58,22 @@
<!-- this profile is active by default -->
<activeByDefault>true</activeByDefault>
</activation>
-
<modules>
<module>indexer-core</module>
</modules>
</profile>
+ <profile>
+ <id>indexer-aws</id>
+ <activation>
+ <property>
+ <name>provider</name>
+ <value>indexer-aws</value>
+ </property>
+ </activation>
+ <modules>
+ <module>provider/indexer-aws</module>
+ </modules>
+ </profile>
<profile>
<id>indexer-azure</id>
<activation>
diff --git a/provider/indexer-aws/CloudFormation/Automated/cache.yml b/provider/indexer-aws/CloudFormation/Automated/cache.yml
new file mode 100644
index 0000000000000000000000000000000000000000..dcdc9a87f34a1e391eed3e3a178b79ada9904c0c
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/cache.yml
@@ -0,0 +1,205 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for the tenant info database for OSDU.
+ It creates the DynamoDB table and the API Gateway endpoints.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ApplicationName:
+ Description: >
+ The name of the application, which will be used to generate the ECS cluster name.
+ It will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-storage
+
+ CacheName:
+ Description: The name of the cache cluster. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: cache
+
+ CacheEngine:
+ Description: Which caching platform to use. Can be set to 'redis' or 'memcached'.
+ Type: String
+ AllowedValues:
+ - redis
+ - memcached
+ ConstraintDescription: Can only be "redis" or "memcached"
+ Default: redis
+
+ NodeInstanceType:
+ Description: The instance type for redis cache nodes.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: cache.t2.micro
+ AllowedValues:
+ - cache.m5.large
+ - cache.m5.xlarge
+ - cache.m5.2xlarge
+ - cache.m5.4xlarge
+ - cache.m5.12xlarge
+ - cache.m5.24xlarge
+ - cache.m4.large
+ - cache.m4.xlarge
+ - cache.m4.2xlarge
+ - cache.m4.4xlarge
+ - cache.m4.10xlarge
+ - cache.t2.micro
+ - cache.t2.small
+ - cache.t2.medium
+ - cache.c1.xlarge
+ - cache.r5.large
+ - cache.r5.xlarge
+ - cache.r5.2xlarge
+ - cache.r5.4xlarge
+ - cache.r5.12xlarge
+ - cache.r5.24xlarge
+ - cache.r4.large
+ - cache.r4.xlarge
+ - cache.r4.2xlarge
+ - cache.r4.4xlarge
+ - cache.r4.8xlarge
+ - cache.r4.16xlarge
+ Type: String
+
+ NumberOfCacheNodes:
+ Description: An integer value specifying the number of node in the redis cache.
+ Type: Number
+ Default: 1
+ MinValue: 1
+ MaxValue: 128
+
+Conditions:
+ IsSingleNode: !Equals [ !Ref NumberOfCacheNodes, 1 ]
+
+ IsClustered: !Not [Condition: IsSingleNode]
+
+ IsMemcached: !Equals [ !Ref CacheEngine, memcached ]
+
+ IsRedis: !Equals [ !Ref CacheEngine, redis ]
+
+Resources:
+ ElastiCacheVpcSecurityGroup:
+ Type: AWS::EC2::SecurityGroup
+ Properties:
+ GroupName: !Sub "${Environment}-${CacheName}-sg"
+ GroupDescription: "This is the security group that all of our ElastiCache cluster will be placed into."
+ VpcId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-VPC"
+
+ ElastiCacheVpcSecurityGroupCodeBuildIngress:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref ElastiCacheVpcSecurityGroup
+ IpProtocol: tcp
+ FromPort: "6379"
+ ToPort: "6379"
+ SourceSecurityGroupId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
+
+ ElastiCacheVpcSecurityGroupECSIngress:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref ElastiCacheVpcSecurityGroup
+ IpProtocol: tcp
+ FromPort: "6379"
+ ToPort: "6379"
+ SourceSecurityGroupId:
+ Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
+
+ ElastiCacheSubnetGroup:
+ Type: 'AWS::ElastiCache::SubnetGroup'
+ Properties:
+ CacheSubnetGroupName: !Sub ${Environment}-${CacheName}-SubnetGroup
+ Description: Redis cache VPC subnet group.
+ SubnetIds:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
+
+ ElastiCacheCluster:
+ Type: 'AWS::ElastiCache::CacheCluster'
+ DependsOn: ElastiCacheSubnetGroup
+ Properties:
+ AutoMinorVersionUpgrade: 'false'
+ AZMode: single-az # this parameter only affects Memcached clusters
+ Engine: !Ref CacheEngine
+ CacheNodeType: !Ref NodeInstanceType
+ NumCacheNodes: !Ref NumberOfCacheNodes
+ ClusterName: !Sub ${Environment}-${CacheName}
+ CacheSubnetGroupName: !Ref ElastiCacheSubnetGroup
+ VpcSecurityGroupIds:
+ - Ref: ElastiCacheVpcSecurityGroup
+
+Outputs:
+ # Redis (cluster mode disabled) replication groups don't have this attribute.
+ # Therefore, Fn::GetAtt returns a value for this attribute only if the replication
+ # group is clustered. Otherwise, Fn::GetAtt fails.
+ ElastiCacheConfigurationEndpointUrl:
+ Description: The configuration endpoint URL of the cache node.
+ Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Address
+ Condition: IsClustered
+ Export:
+ Name: !Sub ${Environment}-${CacheName}-ElastiCacheConfigurationEndpointUrl
+
+ # This output is only applicable if the cache engine is set to Memcached
+ MemcachedConfigurationEndpointPort:
+ Description: The Memcached configuration endpoint port of the cache node.
+ Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Port
+ Condition: IsMemcached
+ Export:
+ Name: !Sub ${Environment}-${CacheName}-MemcachedConfigurationEndpointPort
+
+ # This output is only applicable if the cache engine is set to Redis
+ RedisEndpointAddress:
+ Description: The Redis endpoint address of the cache.
+ Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Address
+ Condition: IsRedis
+ Export:
+ Name: !Sub ${Environment}-${CacheName}-RedisEndpointAddress
+
+ # This output is only applicable if the cache engine is set to Redis
+ RedisEndpointPort:
+ Description: The Redis endpoint port of the cache.
+ Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Port
+ Condition: IsRedis
+ Export:
+ Name: !Sub ${Environment}-${CacheName}-RedisEndpointPort
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3e2787b66b85926b0153ad194a33191ff29b8f0b
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
@@ -0,0 +1,568 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for the ECS cluster the application will
+ be deployed into. Will create the CodeDeploy application, the ECR repository, and the ECS cluster.
+ This is separated from the rest of the ECS resources in order to avoid a circular dependency.
+ Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
+ the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
+ the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
+ whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
+ add or remove ElastiCache clusters.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ApplicationName:
+ Description: >
+ The name of the application, which will be used to generate the ECS cluster name.
+ It will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-indexer
+
+ KeyName:
+ Description: >
+ Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
+ be created through CloudFormation, but instead must be uploaded through the AWS Console.
+ Type: AWS::EC2::KeyPair::KeyName
+ Default: ecs_indexer_key
+
+ DesiredCapacity:
+ Description: The default number of instances to launch in the ECS cluster.
+ Type: Number
+ Default: '1'
+
+ MaxSize:
+ Description: Maximum number of instances that can be launched in the ECS cluster.
+ Type: Number
+ Default: '1'
+
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ Default: t3.large
+ AllowedValues:
+ - m5.large
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ - m5.12xlarge
+ - m5.16xlarge
+ - m5.24xlarge
+ - m4.large
+ - m4.xlarge
+ - m4.2xlarge
+ - m4.4xlarge
+ - m4.10xlarge
+ - m4.16xlarge
+ - t3.nano
+ - t3.micro
+ - t3.small
+ - t3.medium
+ - t3.large
+ - t3.xlarge
+ - t3.2xlarge
+ - c5.large
+ - c5.xlarge
+ - c5.2xlarge
+ - c5.4xlarge
+ - c5.12xlarge
+ - c5.16xlarge
+ - c5.24xlarge
+ - r5.large
+ - r5.xlarge
+ - r5.2xlarge
+ - r5.4xlarge
+ - r5.12xlarge
+ - r5.24xlarge
+ - r4.large
+ - r4.xlarge
+ - r4.2xlarge
+ - r4.4xlarge
+ - r4.8xlarge
+ - r4.16xlarge
+ - i3.large
+ - i3.xlarge
+ - i3.2xlarge
+ - i3.4xlarge
+ - i3.10xlarge
+ - i3.16xlarge
+ - x1e.xlarge
+ - x1e.2xlarge
+ - x1e.4xlarge
+ - x1e.8xlarge
+ - x1e.16xlarge
+ - x1e.32xlarge
+ ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
+
+ SchemaCacheName:
+ Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerSchemaCache
+
+ IndexCacheName:
+ Description: The name of the cache cluster for the index cache. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerIndexCache
+
+ ECSPort:
+ Description: The port that the ECS Service will listen on.
+ Type: Number
+ Default: 80
+ MinValue: 1
+ MaxValue: 65535
+
+ SNSTopicName:
+ Description: >-
+ The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
+ Will be prefixed with the environment name.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer-messages
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+
+ ECSCPUAllocation:
+ Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
+ Type: Number
+ Default: 1024
+ MinValue: 10
+ MaxValue: 65535
+
+ ECSMemoryAllocation:
+ Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
+ Type: Number
+ Default: 2048
+ MinValue: 256
+ MaxValue: 131072
+
+Mappings:
+ # This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
+ # It will need to be periodically updated as new versions are released by Amazon.
+ # The latest ECS-optimized AMI IDs can be found here: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
+ # The mapping is used to input the correct AMI ID based on the region the instance is being spun up in.
+ AWSRegionToAMI:
+ us-east-1:
+ AMIID: ami-097e3d1cdb541f43e
+ us-east-2:
+ AMIID: ami-0fbd313043845c4f2
+ us-west-1:
+ AMIID: ami-03d7632ea0ab75eaa
+ us-west-2:
+ AMIID: ami-0fb71e703258ab7eb
+ eu-north-1:
+ AMIID: ami-0f8edbbca6bac13a6
+ eu-west-1:
+ AMIID: ami-0bf45a5f4ab05b949
+ eu-west-2:
+ AMIID: ami-0393b5f363fbd613a
+ eu-west-3:
+ AMIID: ami-03490ca40775a62f0
+ eu-central-1:
+ AMIID: ami-074dc9dd588b6ea52
+ ap-northeast-1:
+ AMIID: ami-0934e28fe3e390537
+ ap-northeast-2:
+ AMIID: ami-0fa5d85859452a178
+ ap-south-1:
+ AMIID: ami-0312d67ff59a3db34
+ ap-southeast-1:
+ AMIID: ami-01f07b3fa86406c96
+ ap-southeast-2:
+ AMIID: ami-07610e278b1ddf331
+ ca-central-1:
+ AMIID: ami-0057d82f917a17334
+ sa-east-1:
+ AMIID: ami-0c947c117562538ee
+
+Resources:
+ CodeDeployApplication:
+ Type: AWS::CodeDeploy::Application
+ Properties:
+ ApplicationName: !Sub ${Environment}-${ApplicationName}-code-deploy
+ ComputePlatform: ECS
+
+ ECRRepository:
+ Type: AWS::ECR::Repository
+ Properties:
+ RepositoryName: !Sub ${Environment}-${ApplicationName}-repository
+ RepositoryPolicyText:
+ Version: "2012-10-17"
+ Statement:
+ - Sid: AllowPushPull
+ Effect: Allow
+ Principal:
+ AWS:
+ - !Sub arn:aws:iam::${AWS::AccountId}:root
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-CodeBuildRoleArn"
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-CFNRoleArn"
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-PipelineRoleArn"
+ Service:
+ - codebuild.amazonaws.com
+ Action:
+ - "ecr:GetDownloadUrlForLayer"
+ - "ecr:BatchGetImage"
+ - "ecr:BatchCheckLayerAvailability"
+ - "ecr:PutImage"
+ - "ecr:InitiateLayerUpload"
+ - "ecr:UploadLayerPart"
+ - "ecr:CompleteLayerUpload"
+
+ ApplicationECSCluster:
+ Type: AWS::ECS::Cluster
+ Properties:
+ ClusterName: !Sub ${Environment}-${ApplicationName}-cluster
+ Tags:
+ - Key: Environment
+ Value: !Ref Environment
+
+ CloudWatchLogsGroup:
+ Type: AWS::Logs::LogGroup
+ Properties:
+ LogGroupName: !Join ['-', [ECSLogGroup, !Ref 'ApplicationName']]
+ RetentionInDays: 365
+
+ TaskDefinition:
+ Type: AWS::ECS::TaskDefinition
+ Properties:
+ Family: !Join ['', [!Ref 'AWS::StackName', -, !Ref 'ApplicationName']]
+ ContainerDefinitions:
+ - Name: !Ref 'ApplicationName'
+ Cpu: !Ref ECSCPUAllocation
+ Essential: 'true'
+ Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Environment}-${ApplicationName}-repository:latest
+ Memory: !Ref ECSMemoryAllocation
+ LogConfiguration:
+ LogDriver: awslogs
+ Options:
+ awslogs-group: !Ref 'CloudWatchLogsGroup'
+ awslogs-region: !Ref 'AWS::Region'
+ awslogs-stream-prefix: !Ref 'ApplicationName'
+ MountPoints:
+ - ContainerPath: /root/.m2
+ SourceVolume: docker-volume
+ PortMappings:
+ - ContainerPort: !Ref ECSPort
+ Environment:
+ - Name: AWS_ACCESS_KEY_ID
+ Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:access_key}}'
+ - Name: AWS_SECRET_KEY
+ Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:secret_key}}'
+ - Name: ENVIRONMENT
+ Value: !Ref Environment
+ - Name: VSTS_FEED_TOKEN
+ Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
+ - Name: CACHE_CLUSTER_SCHEMA_ENDPOINT
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-${SchemaCacheName}-RedisEndpointAddress"
+ - Name: CACHE_CLUSTER_SCHEMA_PORT
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-${SchemaCacheName}-RedisEndpointPort"
+ - Name: CACHE_CLUSTER_INDEX_ENDPOINT
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-${IndexCacheName}-RedisEndpointAddress"
+ - Name: CACHE_CLUSTER_INDEX_PORT
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-${IndexCacheName}-RedisEndpointPort"
+ - Name: APPLICATION_PORT
+ Value: !Ref ECSPort
+ - Name: AWS_REGION
+ Value: !Ref 'AWS::Region'
+ - Name: AWS_ACCOUNT_ID
+ Value: !Ref 'AWS::AccountId'
+ - Name: SNS_TOPIC_NAME
+ Value: !Ref SNSTopicName
+ - Name: JAVA_HEAP_MEMORY
+ Value: !Ref ECSMemoryAllocation
+ Volumes:
+ - Name: docker-volume
+
+ ECSALB:
+ Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+ Properties:
+ Name: !Sub ECSALB-${ApplicationName}
+ Scheme: internet-facing
+ LoadBalancerAttributes:
+ - Key: idle_timeout.timeout_seconds
+ Value: '30'
+ Subnets:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
+ SecurityGroups:
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
+
+ ALBListener:
+ Type: AWS::ElasticLoadBalancingV2::Listener
+ DependsOn: ECSServiceRole
+ Properties:
+ DefaultActions:
+ - Type: forward
+ TargetGroupArn: !Ref 'ECSTargetGroup'
+ LoadBalancerArn: !Ref 'ECSALB'
+ Port: !Ref ECSPort
+ Protocol: HTTP
+
+ ECSALBPrimaryListenerRule:
+ Type: AWS::ElasticLoadBalancingV2::ListenerRule
+ DependsOn: ALBListener
+ Properties:
+ Actions:
+ - Type: forward
+ TargetGroupArn: !Ref 'ECSTargetGroup'
+ Conditions:
+ - Field: path-pattern
+ Values: [/]
+ ListenerArn: !Ref 'ALBListener'
+ Priority: 1
+
+ ECSTargetGroup:
+ Type: AWS::ElasticLoadBalancingV2::TargetGroup
+ DependsOn: ECSALB
+ Properties:
+ HealthCheckIntervalSeconds: 120
+ HealthCheckPath: /api/indexer/v2/liveness_check
+ HealthCheckProtocol: HTTP
+ HealthCheckTimeoutSeconds: 5
+ HealthyThresholdCount: 2
+ Name: !Sub ECSTargetGroup-${ApplicationName}
+ Port: !Ref ECSPort
+ Protocol: HTTP
+ UnhealthyThresholdCount: 2
+ VpcId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-VPC"
+
+ ECSAutoScalingGroup:
+ Type: AWS::AutoScaling::AutoScalingGroup
+ Properties:
+ VPCZoneIdentifier:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
+ LaunchConfigurationName: !Ref 'ContainerInstances'
+ MinSize: '1'
+ MaxSize: !Ref 'MaxSize'
+ DesiredCapacity: !Ref 'DesiredCapacity'
+ CreationPolicy:
+ ResourceSignal:
+ Timeout: PT15M
+ UpdatePolicy:
+ AutoScalingReplacingUpdate:
+ WillReplace: 'true'
+
+ ContainerInstances:
+ Type: AWS::AutoScaling::LaunchConfiguration
+ Properties:
+ ImageId: !FindInMap [AWSRegionToAMI, !Ref 'AWS::Region', AMIID]
+ SecurityGroups:
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
+ InstanceType: !Ref 'InstanceType'
+ IamInstanceProfile: !Ref 'EC2InstanceProfile'
+ KeyName: !Ref 'KeyName'
+ UserData:
+ Fn::Base64: !Sub |
+ #!/bin/bash -xe
+ echo ECS_CLUSTER=${ApplicationECSCluster} >> /etc/ecs/ecs.config
+ yum install -y aws-cfn-bootstrap
+ /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
+
+ Service:
+ Type: AWS::ECS::Service
+ DependsOn: ALBListener
+ Properties:
+ Cluster: !Ref 'ApplicationECSCluster'
+ DesiredCount: '1'
+ LoadBalancers:
+ - ContainerName: !Ref 'ApplicationName'
+ ContainerPort: !Ref ECSPort
+ TargetGroupArn: !Ref 'ECSTargetGroup'
+ Role: !Ref 'ECSServiceRole'
+ TaskDefinition: !Ref 'TaskDefinition'
+
+ ECSServiceRole:
+ Type: AWS::IAM::Role
+ Properties:
+ AssumeRolePolicyDocument:
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: [ecs.amazonaws.com]
+ Action: ['sts:AssumeRole']
+ Path: /
+ Policies:
+ - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
+ PolicyDocument:
+ Statement:
+ - Effect: Allow
+ Action: ['elasticloadbalancing:DeregisterInstancesFromLoadBalancer', 'elasticloadbalancing:DeregisterTargets',
+ 'elasticloadbalancing:Describe*', 'elasticloadbalancing:RegisterInstancesWithLoadBalancer',
+ 'elasticloadbalancing:RegisterTargets', 'ec2:Describe*', 'ec2:AuthorizeSecurityGroupIngress']
+ Resource: '*'
+
+ ServiceScalingTarget:
+ Type: AWS::ApplicationAutoScaling::ScalableTarget
+ DependsOn: Service
+ Properties:
+ MaxCapacity: 2
+ MinCapacity: 1
+ ResourceId: !Join ['', [service/, !Ref 'ApplicationECSCluster', /, !GetAtt [Service, Name]]]
+ RoleARN: !GetAtt [AutoscalingRole, Arn]
+ ScalableDimension: ecs:service:DesiredCount
+ ServiceNamespace: ecs
+
+ ServiceScalingPolicy:
+ Type: AWS::ApplicationAutoScaling::ScalingPolicy
+ Properties:
+ PolicyName: !Sub ScalingPolicy-${ApplicationName}
+ PolicyType: StepScaling
+ ScalingTargetId: !Ref 'ServiceScalingTarget'
+ StepScalingPolicyConfiguration:
+ AdjustmentType: PercentChangeInCapacity
+ Cooldown: 60
+ MetricAggregationType: Average
+ StepAdjustments:
+ - MetricIntervalLowerBound: 0
+ ScalingAdjustment: 200
+
+ ALB500sAlarmScaleUp:
+ Type: AWS::CloudWatch::Alarm
+ Properties:
+ EvaluationPeriods: '1'
+ Statistic: Average
+ Threshold: '10'
+ AlarmDescription: Alarm triggering ECS to scale up if our ALB generates too many HTTP 500 errors.
+ Period: '60'
+ AlarmActions: [!Ref 'ServiceScalingPolicy']
+ Namespace: AWS/ApplicationELB
+ Dimensions:
+ - Name: LoadBalancer
+ Value: !GetAtt
+ - ECSALB
+ - LoadBalancerFullName
+ ComparisonOperator: GreaterThanThreshold
+ MetricName: HTTPCode_ELB_5XX_Count
+
+ EC2Role:
+ Type: AWS::IAM::Role
+ Properties:
+ AssumeRolePolicyDocument:
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: [ec2.amazonaws.com]
+ Action: ['sts:AssumeRole']
+ Path: /
+ Policies:
+ - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
+ PolicyDocument:
+ Statement:
+ - Effect: Allow
+ Action: ['ecs:CreateCluster', 'ecs:DeregisterContainerInstance', 'ecs:DiscoverPollEndpoint',
+ 'ecs:Poll', 'ecs:RegisterContainerInstance', 'ecs:StartTelemetrySession',
+ 'ecs:Submit*', 'logs:CreateLogStream', 'logs:PutLogEvents', 'ecr:*']
+ Resource: '*'
+
+ AutoscalingRole:
+ Type: AWS::IAM::Role
+ Properties:
+ AssumeRolePolicyDocument:
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: [application-autoscaling.amazonaws.com]
+ Action: ['sts:AssumeRole']
+ Path: /
+ Policies:
+ - PolicyName: !Sub ${Environment}-${ApplicationName}-service-autoscaling
+ PolicyDocument:
+ Statement:
+ - Effect: Allow
+ Action: ['application-autoscaling:*', 'cloudwatch:DescribeAlarms', 'cloudwatch:PutMetricAlarm',
+ 'ecs:DescribeServices', 'ecs:UpdateService']
+ Resource: '*'
+
+ EC2InstanceProfile:
+ Type: AWS::IAM::InstanceProfile
+ Properties:
+ Path: /
+ Roles: [!Ref 'EC2Role']
+
+Outputs:
+ ApplicationECSClusterArn:
+ Description: The ARN of the application's ECS cluster.
+ Value: !GetAtt ApplicationECSCluster.Arn
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsClusterArn
+
+ ApplicationECSClusterName:
+ Description: The logical name of the application's ECS cluster.
+ Value: !Ref ApplicationECSCluster
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsClusterName
+
+ ECSServiceArn:
+ Description: The ARN of the Indexer Service service in the ECS cluster.
+ Value: !Ref 'Service'
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsServiceArn
+
+ ECSServiceName:
+ Description: The name of the Legal Service service in the ECS cluster.
+ Value: !GetAtt Service.Name
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsServiceName
+
+ ECSALBUrl:
+ Description: The Indexer Service ALB DNS URL.
+ Value: !Join ['', [!GetAtt [ECSALB, DNSName]]]
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
+
+ TaskDefinitionArn:
+ Description: The ARN of the Indexer Service ECS task definition.
+ Value: !Ref 'TaskDefinition'
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsTaskDefinitionArn
+
+ IndexerEC2RoleArn:
+ Description: The ARN of the application's EC2 role.
+ Value: !GetAtt EC2Role.Arn
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EC2RoleArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
new file mode 100644
index 0000000000000000000000000000000000000000..cd3baa306cf870d1663747520c8823e542543214
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
@@ -0,0 +1,120 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the network resources used for the ECS cluster the application will
+ be deployed into. This is separated from the rest of the ECS resources in order to avoid a circular dependency.
+ Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
+ the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
+ the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
+ whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
+ add or remove ElastiCache clusters.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ApplicationName:
+ Description: >
+ The name of the application, which will be used to generate the ECS cluster name.
+ It will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-indexer
+
+ ECSPort:
+ Description: The port that the ECS Service will listen on.
+ Type: Number
+ Default: 80
+ MinValue: 1
+ MaxValue: 65535
+
+Resources:
+ ECSSecurityGroup:
+ Type: AWS::EC2::SecurityGroup
+ Properties:
+ GroupName: !Sub "${Environment}-${ApplicationName}-sg"
+ GroupDescription: indexer Service ECS Security Group
+ VpcId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-VPC"
+
+ # Public access to ECS Listening Port
+ ECSSecurityGroupECSListenerInbound:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: !Ref ECSPort
+ ToPort: !Ref ECSPort
+ CidrIp: 0.0.0.0/0
+
+ # Public access to port 443
+ ECSSecurityGroupHTTPSInbound:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: '443'
+ ToPort: '443'
+ CidrIp: 0.0.0.0/0
+
+ # Public access to port 8080
+ ECSSecurityGroupHTTPAltInbound:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: '8080'
+ ToPort: '8080'
+ CidrIp: 0.0.0.0/0
+
+ # Public access to port 8443
+ ECSSecurityGroupHTTPSAltInbound:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: '8443'
+ ToPort: '8443'
+ CidrIp: 0.0.0.0/0
+
+ # SSH access for instances in our VPC's jump box subnet group (coming soon – will be part of the Util CFN)
+ ECSSecurityGroupSSHInbound:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: '22'
+ ToPort: '22'
+ CidrIp: 0.0.0.0/0
+
+ # Open Application Load Balancer port range to itself
+ ECSSecurityGroupALBports:
+ Type: AWS::EC2::SecurityGroupIngress
+ Properties:
+ GroupId: !Ref 'ECSSecurityGroup'
+ IpProtocol: tcp
+ FromPort: '31000'
+ ToPort: '61000'
+ SourceSecurityGroupId: !Ref 'ECSSecurityGroup'
+
+Outputs:
+ EcsNetworkSecurityGroupId:
+ Description: The ID of the indexer Service ECS EC2 security group.
+ Value: !Ref 'ECSSecurityGroup'
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId
diff --git a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
new file mode 100644
index 0000000000000000000000000000000000000000..00ef39a89b1c2496f654c32a620fe485a662569e
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
@@ -0,0 +1,230 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for the tenant info database for OSDU.
+ It creates the DynamoDB table and the API Gateway endpoints.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ElasticsearchDomainName:
+ Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer
+
+ ElasticsearchNodeInstanceType:
+ Description: The instance type for the main Elasticsearch nodes.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ DedicatedMasterInstanceType:
+ Description: >
+ The instance type for the dedicated master nodes. These nodes perform cluster management
+ tasks, but doesn't hold data or respond to data upload requests.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ NumberOfElasticsearchNodes:
+ Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+ Type: Number
+ Default: 2
+ MinValue: 1
+ MaxValue: 40
+
+ NumberOfDedicatedMasterNodes:
+ Description: An integer value specifying the number of dedicated master nodes.
+ Type: Number
+ Default: 2
+ MinValue: 2
+ MaxValue: 5
+
+ ZoneAwarenessEnabled:
+ Description: >
+ When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+ index shards that belong to a cluster across multiple AZs in the deployment region.
+ Type: String
+ AllowedValues:
+ - true
+ - false
+ Default: false
+
+ ElasticsearchVersion:
+ Description: >
+ The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+ that an update requires a full replacement of the Elasticsearch cluster.
+ Type: String
+ AllowedValues:
+ - 1.5
+ - 2.3
+ - 5.1
+ - 5.3
+ - 5.5
+ - 5.6
+ - 6.0
+ - 6.2
+ - 6.3
+ - 6.4
+ - 6.5
+ - 6.6
+ - 6.8
+ - 6.8
+ - 7.1
+ Default: 6.8
+
+ EBSVolumeSize:
+ Description: >
+ The size of the EBS volume (per instance; total cluster size = EBS volume size x Instance count)
+ Maximum size varies by instance type, from 35GiB for t2 instances, up to 12TiB for r5.12xlarge.
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 12000
+
+Resources:
+ ElasticsearchDomain:
+ Type: AWS::Elasticsearch::Domain
+ Properties:
+ DomainName: !Sub ${Environment}-${ElasticsearchDomainName}
+ ElasticsearchVersion: !Ref ElasticsearchVersion
+ ElasticsearchClusterConfig:
+ DedicatedMasterEnabled: "true"
+ InstanceCount: !Ref NumberOfElasticsearchNodes
+ ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+ InstanceType: !Ref ElasticsearchNodeInstanceType
+ DedicatedMasterType: !Ref DedicatedMasterInstanceType
+ DedicatedMasterCount: !Ref NumberOfDedicatedMasterNodes
+ EBSOptions:
+ EBSEnabled: true
+ VolumeSize: !Ref EBSVolumeSize
+ VolumeType: "gp2"
+ NodeToNodeEncryptionOptions:
+ Enabled: false
+ SnapshotOptions:
+ AutomatedSnapshotStartHour: "0"
+# AccessPolicies:
+# - Version: "2012-10-17"
+# Statement:
+# - Effect: "Allow"
+# Principal:
+# AWS:
+# - !Sub arn:aws:iam::${AWS::AccountId}:root
+# - Fn::ImportValue:
+# !Sub "${Environment}-IndexerServiceIamUserArn"
+# # TODO: need to create cognito user and identity pool and link it to principal for dynamic creation
+# - "arn:aws:iam::888733619319:role/Cognito_osduelasticsearchAuth_Role"
+# Action:
+# - "es:*"
+# - 'es:ESHttp*'
+# - 'cognito-identity:*'
+# - 'cognito-idp:*'
+# - 'sts:AssumeRole'
+# Resource: !Sub arn:aws:es:us-east-1:846973539254:domain/${Environment}-${ElasticsearchDomainName}/*
+# - "Version": "2012-10-17"
+# "Statement":
+# - "Effect": "Allow"
+# "Action":
+# - "iam:PassRole"
+# "Resource": "arn:aws:iam::888733619319:role/service-role/CognitoAccessForAmazonES"
+ AdvancedOptions:
+ rest.action.multi.allow_explicit_index: "true"
+ Tags:
+ -
+ Key: "Environment"
+ Value: !Ref Environment
+
+Outputs:
+ # Elasticsearch domain ARN
+ ElasticsearchDomainArn:
+ Description: The ARN of the Elasticsearch domain.
+ Value: !GetAtt ElasticsearchDomain.DomainArn
+ Export:
+ Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainArn
+
+ # Elasticsearch domain endpoint
+ ElasticsearchDomainEndpoint:
+ Description: The endpoint URL of the Elasticsearch domain.
+ Value: !GetAtt ElasticsearchDomain.DomainEndpoint
+ Export:
+ Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint
diff --git a/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml b/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3d5c9f144d52070bfebb81eacc4f32ba4b193f31
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
@@ -0,0 +1,114 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for application SDK access for OSDU services.
+ It creates the IAM account, access keys, and optional key rotation.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ IndexerServiceIamUsername:
+ Description: The username of the service user for the OS Indexer Service.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Type: String
+ Default: service-user-os-indexer
+ MinLength: '1'
+ MaxLength: '64'
+
+ IndexerServiceIamKeyRotationSerial:
+ Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
+ Type: Number
+ Default: 1
+
+Resources:
+ IndexerServiceIamUser:
+ Type: AWS::IAM::User
+ Properties:
+ Policies:
+ - PolicyName: !Sub ${Environment}-IndexerServiceUserPolicy
+ PolicyDocument:
+ Version: '2012-10-17'
+ Statement:
+ -
+ Action:
+ - 's3:*'
+ - 'sns:*'
+ - 'sqs:*'
+ - 'dynamodb:*'
+ - 'logs:*'
+ - 'cloudwatch:*'
+ - 'es:*'
+ - 'cognito-identity:*'
+ - 'cognito-idp:*'
+ - 'sts:AssumeRole'
+ - "iam:*"
+ Effect: Allow
+ Resource: '*'
+ UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
+
+ IndexerServiceIamUserAccessKey:
+ Type: AWS::IAM::AccessKey
+ DependsOn: IndexerServiceIamUser
+ Properties:
+ Serial: !Ref IndexerServiceIamKeyRotationSerial # this value can only ever be incremented, and an increase in value results in a rotation of the user's access key
+ Status: Active
+ UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
+
+ IAMCredentialsSecret:
+ Type: 'AWS::SecretsManager::Secret'
+ Properties:
+ Name: !Sub ${Environment}-IndexerServiceIamCredentials
+ Description: The IAM service account credentials for the search service.
+ SecretString:
+ Fn::Sub:
+ - '{"access_key":"${AccessKey}","secret_key":"${SecretKey}"}'
+ - {AccessKey: !Ref IndexerServiceIamUserAccessKey, SecretKey: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey}
+ Tags:
+ - Key: Environment
+ Value: !Ref Environment
+
+Outputs:
+ IndexerServiceIamUserAccessKeyId:
+ Description: The access key ID for the service user for the Schema Repository.
+ Value: !Ref IndexerServiceIamUserAccessKey
+ Export:
+ Name: !Sub ${Environment}-IndexerServiceIamUserAccessKeyId
+
+ IndexerServiceIamUserSecretAccessKey:
+ Description: The secret access key for the service user for the Schema Repository.
+ Value: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey
+ Export:
+ Name: !Sub ${Environment}-IndexerServiceIamUserSecretAccessKey
+
+ IndexerServiceIamUserArn:
+ Description: The ARN of the service IAM user account.
+ Value: !GetAtt IndexerServiceIamUser.Arn
+ Export:
+ Name: !Sub ${Environment}-IndexerServiceIamUserArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml b/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f90f9188963232b89978dde65f51597fb79f98e9
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
@@ -0,0 +1,108 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for the sending messages to topic and queues to receive the
+ messages for OSDU's indexer service. It creates the SNS Topic and the corresponding SQS Queues with their associated policies.
+
+Parameters:
+ Environment:
+ Description: an environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ SNSTopicName:
+ Description: >-
+ The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
+ Will be prefixed with the environment name.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer-messages
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+
+ SQSQueueName:
+ Description: >-
+ The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
+ Will be prefixed with the environment name.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer-queue
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+
+Resources:
+ OSDUIndexerSNSTopic:
+ Type: 'AWS::SNS::Topic'
+ Properties:
+ DisplayName: !Sub ${Environment}-${SNSTopicName}
+ TopicName: !Sub ${Environment}-${SNSTopicName}
+ Subscription:
+ - Endpoint:
+ Fn::GetAtt:
+ - OSDUIndexerSQSQueue
+ - Arn
+ Protocol: sqs
+
+ OSDUIndexerSQSQueue:
+ Type: AWS::SQS::Queue
+ Properties:
+ QueueName: !Sub ${Environment}-${SQSQueueName}
+
+ OSDUQueuePolicy:
+ Type: AWS::SQS::QueuePolicy
+ Properties:
+ PolicyDocument:
+ Version: "2012-10-17"
+ Id: OSDUQueuePolicy
+ Statement:
+ - Sid: Allow-SendMessage-To-Queues-From-SNS-Topic
+ Effect: Allow
+ Principal: "*"
+ Action:
+ - sqs:SendMessage
+ - sqs:ReceiveMessage
+ Resource: "*"
+ Condition:
+ ArnEquals:
+ aws:SourceArn:
+ Ref: OSDUIndexerSNSTopic
+ Queues:
+ - Ref: OSDUIndexerSQSQueue
+
+Outputs:
+ OSDUIndexerSNSTopicTopicName:
+ Value: !Sub ${Environment}-${SNSTopicName}
+ Description: Topic Name of the Indexer Service Message Bus SNS Topic
+ Export:
+ Name: !Sub ${Environment}-OSDUIndexerSNSTopic
+
+ OSDUIndexerSQSQueueName:
+ Value: !Sub ${Environment}-${SQSQueueName}
+ Description: Queue Name of Subscribed Indexer Service Message Bus SQS Queue
+ Export:
+ Name: !Sub ${Environment}-OSDUIndexerSQSQueue
diff --git a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
new file mode 100644
index 0000000000000000000000000000000000000000..cfaad181934f70d70b9ce9213c9ab0f6b3175c1f
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
@@ -0,0 +1,680 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: >
+ This CloudFormation script creates the deployment pipeline for OSDU's indexer
+ service. The CodePipeline should automatically trigger whenever commits are
+ made on the tracked branch. The start and end of the CodePipeline should
+ trigger a SNS alert to keep track of when the deployment has started and when
+ it finishes.
+Parameters:
+ Environment:
+ Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ Default: dev
+
+ DeploymentRegion:
+ Description: The AWS region to deploy the application to. The default is us-east-1.
+ Type: String
+ Default: us-east-1
+
+ SNSNotificationEmail:
+ Description: The email address to send SNS notifications about the build to.
+ Type: String
+ Default: barclay.walsh@parivedasolutions.com
+
+ CodeCommitRepositoryName:
+ Description: >-
+ The name of the Code Commit Repository that the CodePipeline source is
+ connected to.
+ Type: String
+ Default: os-indexer
+
+ CodeCommitBranchName:
+ Description: >-
+ The name of the Code Commit branch that the CodePipeline source is
+ connected to.
+ Type: String
+ Default: dev
+
+ MasterStackName:
+ Description: The name of the master stack that is being deployed by the CodePipeline.
+ Type: String
+ Default: os-indexer-master-stack
+
+ MasterTemplateName:
+ Description: >-
+ The name of the master template that is called when creating the master
+ stack.
+ Type: String
+ Default: provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
+
+ SchemaCacheName:
+ Description: The name of the cache cluster for the schema cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerSchemaCache
+
+ IndexCacheName:
+ Description: The name of the cache cluster for the index cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerIndexCache
+
+Resources:
+ S3BucketCloudFormation:
+ Type: 'AWS::S3::Bucket'
+ DeletionPolicy: Delete
+ Properties:
+ BucketName: !Sub '${Environment}-os-indexer-cloudformation-scripts'
+ CloudFormationS3BucketPolicy:
+ Type: 'AWS::S3::BucketPolicy'
+ Properties:
+ Bucket: !Ref S3BucketCloudFormation
+ PolicyDocument:
+ Statement:
+ - Action:
+ - 's3:*'
+ Effect: Allow
+ Resource:
+ - !Sub 'arn:aws:s3:::${S3BucketCloudFormation}'
+ - !Sub 'arn:aws:s3:::${S3BucketCloudFormation}/*'
+ Principal:
+ AWS:
+ - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+ - !GetAtt
+ - CodeBuildRole
+ - Arn
+ - !GetAtt
+ - PipelineRole
+ - Arn
+ - !GetAtt
+ - CFNRole
+ - Arn
+ ArtifactStoreBucket:
+ Type: 'AWS::S3::Bucket'
+ DeletionPolicy: Delete
+ Properties:
+ VersioningConfiguration:
+ Status: Enabled
+ ArtifactStoreBucketPolicy:
+ Type: 'AWS::S3::BucketPolicy'
+ Properties:
+ Bucket: !Ref ArtifactStoreBucket
+ PolicyDocument:
+ Statement:
+ - Action:
+ - 's3:*'
+ Effect: Allow
+ Resource:
+ - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}'
+ - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}/*'
+ Principal:
+ AWS:
+ - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+ - !GetAtt
+ - CodeBuildRole
+ - Arn
+ - !GetAtt
+ - PipelineRole
+ - Arn
+ - !GetAtt
+ - CFNRole
+ - Arn
+ CachingBucket:
+ Type: AWS::S3::Bucket
+ DeletionPolicy: Delete
+ Properties:
+ VersioningConfiguration:
+ Status: Enabled
+
+ CachingBucketPolicy:
+ Type: AWS::S3::BucketPolicy
+ Properties:
+ Bucket: !Ref CachingBucket
+ PolicyDocument:
+ Statement:
+ - Action:
+ - s3:*
+ Effect: Allow
+ Resource:
+ - !Sub arn:aws:s3:::${CachingBucket}
+ - !Sub arn:aws:s3:::${CachingBucket}/*
+ Principal:
+ AWS:
+ - !Sub arn:aws:iam::${AWS::AccountId}:root
+ - !GetAtt [CodeBuildRole,Arn]
+ - !GetAtt [PipelineRole,Arn]
+ - !GetAtt [CFNRole,Arn]
+
+ SNSCodePipelineDeploymentFailed:
+ Type: 'AWS::SNS::Topic'
+ Properties:
+ Subscription:
+ - Endpoint: !Ref SNSNotificationEmail
+ Protocol: email
+ TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-Failed'
+ EventRuleCodePipelineFailed:
+ Type: 'AWS::Events::Rule'
+ Properties:
+ Description: Triggered whenever the CodePipeline deployment stage has failed.
+ EventPattern:
+ source:
+ - aws.codepipeline
+ detail-type:
+ - CodePipeline Stage Execution State Change
+ detail:
+ state:
+ - FAILED
+ pipeline:
+ - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
+ Name: !Sub '${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}'
+ Targets:
+ - Arn: !Ref SNSCodePipelineDeploymentFailed
+ Id: Deployment-CodePipeline-Failed
+ InputTransformer:
+ InputPathsMap:
+ pipeline: $.detail.pipeline
+ InputTemplate: '"The Pipeline <pipeline> has failed."'
+ Pipeline:
+ Type: 'AWS::CodePipeline::Pipeline'
+ Properties:
+ ArtifactStore:
+ Location: !Ref ArtifactStoreBucket
+ Type: S3
+ Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
+ RoleArn: !GetAtt
+ - PipelineRole
+ - Arn
+ Stages:
+ - Name: Source
+ Actions:
+ - Name: Source
+ ActionTypeId:
+ Category: Source
+ Owner: AWS
+ Provider: CodeCommit
+ Version: '1'
+ Configuration:
+ BranchName: !Ref CodeCommitBranchName
+ RepositoryName: !Ref CodeCommitRepositoryName
+ OutputArtifacts:
+ - Name: Source
+ RunOrder: '1'
+ - Name: Pre-Deployment-CodeBuild
+ Actions:
+ - Name: Pre-Deployment-CodeBuild
+ ActionTypeId:
+ Category: Build
+ Owner: AWS
+ Provider: CodeBuild
+ Version: '1'
+ InputArtifacts:
+ - Name: Source
+ OutputArtifacts:
+ - Name: Pre-Deployment-CodeBuild
+ Configuration:
+ ProjectName: !Ref PreDeploymentCodeBuild
+ RunOrder: '2'
+ - Name: Deployment
+ Actions:
+ - Name: CloudFormation-Deployment-Master
+ ActionTypeId:
+ Category: Deploy
+ Owner: AWS
+ Provider: CloudFormation
+ Version: '1'
+ InputArtifacts:
+ - Name: Source
+ Configuration:
+ ActionMode: CREATE_UPDATE
+ Capabilities: CAPABILITY_NAMED_IAM
+ RoleArn: !GetAtt
+ - CFNRole
+ - Arn
+ StackName: !Sub '${Environment}-${MasterStackName}'
+ TemplatePath: !Sub 'Source::${MasterTemplateName}'
+ TemplateConfiguration: !Sub >-
+ Source::provider/indexer-aws/CloudFormation/Params/${Environment}.template_configuration.json
+ RunOrder: '3'
+
+ - Name: Post-Deployment-CodeBuild
+ Actions:
+ - Name: Post-Deployment-CodeBuild
+ ActionTypeId:
+ Category: Build
+ Owner: AWS
+ Provider: CodeBuild
+ Version: '1'
+ InputArtifacts:
+ - Name: Source
+ OutputArtifacts:
+ - Name: Post-Deployment-CodeBuild
+ Configuration:
+ ProjectName: !Ref PostDeploymentCodeBuild
+ RunOrder: '4'
+ PreDeploymentCodeBuild:
+ Type: 'AWS::CodeBuild::Project'
+ Properties:
+ Name: !Sub '${Environment}-pre-deployment-codebuild-${CodeCommitRepositoryName}'
+ Description: CodeBuild commands which run prior to the CloudFormation deployment.
+ ServiceRole:
+ 'Fn::GetAtt':
+ - CodeBuildRole
+ - Arn
+ Artifacts:
+ Type: S3
+ Location: !Ref ArtifactStoreBucket
+ Name: !Sub '${Environment}-pre-deployment-codebuild'
+ Environment:
+ Type: LINUX_CONTAINER
+ ComputeType: BUILD_GENERAL1_SMALL
+ Image: aws/codebuild/standard:2.0
+ EnvironmentVariables:
+ - Name: ENVIRONMENT
+ Type: PLAINTEXT
+ Value: !Ref Environment
+ - Name: AWS_ACCOUNT_ID
+ Type: PLAINTEXT
+ Value: !Ref 'AWS::AccountId'
+ - Name: AWS_REGION
+ Type: PLAINTEXT
+ Value: !Ref DeploymentRegion
+ - Name: CFN_S3_BUCKET
+ Type: PLAINTEXT
+ Value: !Sub '${Environment}-os-indexer-cloudformation-scripts'
+ PrivilegedMode: false
+ Source:
+ BuildSpec: ./provider/indexer-aws/buildspec-pre-deploy.yml
+ Location: !Sub >-
+ https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
+ Type: CODECOMMIT
+ TimeoutInMinutes: 15
+
+ PostDeploymentCodeBuild:
+ Type: AWS::CodeBuild::Project
+ Properties:
+ Name: !Sub ${Environment}-post-deployment-codebuild-${CodeCommitRepositoryName}
+ Description: CodeBuild commands which run after the CloudFormation deployment.
+ ServiceRole:
+ Fn::GetAtt: [ CodeBuildRole, Arn ]
+ Artifacts:
+ Type: S3
+ Location: !Ref ArtifactStoreBucket
+ Name: !Sub ${Environment}-post-deployment-codebuild
+ Environment:
+ Type: LINUX_CONTAINER
+ ComputeType: BUILD_GENERAL1_SMALL
+ Image: aws/codebuild/standard:2.0
+ EnvironmentVariables:
+ - Name: ENVIRONMENT
+ Type: PLAINTEXT
+ Value: !Ref Environment
+ - Name: AWS_ACCOUNT_ID
+ Type: PLAINTEXT
+ Value: !Ref AWS::AccountId
+ - Name: AWS_REGION
+ Type: PLAINTEXT
+ Value: !Ref DeploymentRegion
+ - Name: VSTS_FEED_TOKEN
+ Type: PLAINTEXT
+ Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
+ - Name: S3_MAVEN_REPOSITORY
+ Type: PLAINTEXT
+ Value: !Sub ${Environment}-osdu-local-maven-repository
+ - Name: AWS_SECRET_ACCESS_KEY_MAVEN
+ Type: PLAINTEXT
+ Value: '{{resolve:secretsmanager:dev-MavenS3RepositoryIamCredentials:SecretString:secret_key}}'
+ - Name: AWS_ACCESS_KEY_ID_MAVEN
+ Type: PLAINTEXT
+ Value: '{{resolve:secretsmanager:dev-MavenS3RepositoryIamCredentials:SecretString:access_key}}'
+ - Name: IMAGE_TAG
+ Type: PLAINTEXT
+ Value: latest
+ - Name: IMAGE_REPO_NAME
+ Type: PLAINTEXT
+ Value: !Sub ${Environment}-${CodeCommitRepositoryName}-repository
+ - Name: QUEUE_IMAGE_REPO_NAME
+ Type: PLAINTEXT
+ Value: !Sub ${Environment}-${CodeCommitRepositoryName}-queue-repository
+ - Name: SCHEMA_CACHE_NAME
+ Type: PLAINTEXT
+ Value: !Ref SchemaCacheName
+ - Name: INDEX_CACHE_NAME
+ Type: PLAINTEXT
+ Value: !Ref IndexCacheName
+ - Name: APPLICATION_NAME
+ Type: PLAINTEXT
+ Value: !Ref CodeCommitRepositoryName
+ PrivilegedMode: true
+ Source:
+ BuildSpec: ./provider/indexer-aws/buildspec-post-deploy.yml
+ Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
+ Type: CODECOMMIT
+ Cache:
+ Type: S3
+ Location: !Sub ${CachingBucket}/${Environment}
+ TimeoutInMinutes: 15
+ VpcConfig:
+ SecurityGroupIds:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
+ Subnets:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
+ VpcId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-VPC"
+
+ CFNRole:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ AssumeRolePolicyDocument:
+ Statement:
+ - Action:
+ - 'sts:AssumeRole'
+ Effect: Allow
+ Principal:
+ Service:
+ - cloudformation.amazonaws.com
+ Version: 2012-10-17
+ Path: /
+ Policies:
+ - PolicyName: !Sub 'CloudFormationRole-${CodeCommitRepositoryName}'
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Action:
+ - 's3:*'
+ - 'ec2:*'
+ - 'apigateway:*'
+ - 'cloudwatch:*'
+ - 'events:*'
+ - 'logs:*'
+ - 'xray:*'
+ - 'lambda:*'
+ - 'rds:*'
+ - 'codepipeline:*'
+ - 'codecommit:*'
+ - 'cloudformation:*'
+ - 'ecr:*'
+ - 'dynamodb:*'
+ - 'application-autoscaling:*'
+ - 'autoscaling:*'
+ - 'states:*'
+ - 'iam:CreateUser'
+ - 'iam:UpdateUser'
+ - 'iam:DeleteUser'
+ - 'iam:CreateAccessKey'
+ - 'iam:UpdateAccessKey'
+ - 'iam:DeleteAccessKey'
+ - 'iam:Delete*'
+ - 'iam:List*'
+ - 'iam:Get*'
+ - 'iam:Put*'
+ - 'iam:CreateServiceSpecificCredential'
+ - 'iam:DeactivateMFADevice'
+ - 'iam:GenerateServiceLastAccessedDetails'
+ - 'iam:UpdateOpenIDConnectProviderThumbprint'
+ - 'iam:PutRolePolicy'
+ - 'iam:AddRoleToInstanceProfile'
+ - 'iam:SimulateCustomPolicy'
+ - 'iam:UploadSSHPublicKey'
+ - 'iam:UpdateServiceSpecificCredential'
+ - 'iam:RemoveClientIDFromOpenIDConnectProvider'
+ - 'iam:UpdateRoleDescription'
+ - 'iam:UpdateServerCertificate'
+ - 'iam:CreateInstanceProfile'
+ - 'iam:GenerateCredentialReport'
+ - 'iam:UntagRole'
+ - 'iam:PutRolePermissionsBoundary'
+ - 'iam:TagRole'
+ - 'iam:ResetServiceSpecificCredential'
+ - 'iam:PassRole'
+ - 'iam:EnableMFADevice'
+ - 'iam:ResyncMFADevice'
+ - 'iam:UpdateSAMLProvider'
+ - 'iam:CreatePolicy'
+ - 'iam:CreateServiceLinkedRole'
+ - 'iam:UpdateRole'
+ - 'iam:AddClientIDToOpenIDConnectProvider'
+ - 'iam:SetDefaultPolicyVersion'
+ - 'iam:UpdateAssumeRolePolicy'
+ - 'iam:RemoveRoleFromInstanceProfile'
+ - 'iam:CreateRole'
+ - 'iam:AttachRolePolicy'
+ - 'iam:CreateLoginProfile'
+ - 'iam:DetachRolePolicy'
+ - 'iam:AttachUserPolicy'
+ - 'iam:DetachUserPolicy'
+ - 'iam:SimulatePrincipalPolicy'
+ - 'iam:CreateAccountAlias'
+ - 'iam:ChangePassword'
+ - 'iam:UpdateLoginProfile'
+ - 'iam:UpdateAccessKey'
+ - 'iam:UpdateSSHPublicKey'
+ - 'iam:UpdateAccountPasswordPolicy'
+ - 'iam:CreateSAMLProvider'
+ - 'iam:CreateVirtualMFADevice'
+ - 'iam:CreateAccessKey'
+ - 'iam:AddUserToGroup'
+ - 'iam:RemoveUserFromGroup'
+ - 'iam:CreatePolicyVersion'
+ - 'iam:UploadSigningCertificate'
+ - 'iam:TagUser'
+ - 'iam:CreateOpenIDConnectProvider'
+ - 'iam:UploadServerCertificate'
+ - 'iam:UntagUser'
+ - 'iam:UpdateSigningCertificate'
+ - 'sns:*'
+ - 'sqs:*'
+ - 'secretsmanager:*'
+ - 'acm:*'
+ - 'kms:*'
+ - 'cloudfront:*'
+ - 'route53:*'
+ - 'route53domains:*'
+ - 'elasticache:*'
+ - 'es:*'
+ - 'ecr:*'
+ - 'codedeploy:*'
+ - 'elasticloadbalancing:*'
+ - 'ecs:*'
+ - 'servicediscovery:CreatePrivateDnsNamespace'
+ - 'servicediscovery:CreateService'
+ - 'servicediscovery:GetNamespace'
+ - 'servicediscovery:GetOperation'
+ - 'servicediscovery:GetService'
+ - 'servicediscovery:ListNamespaces'
+ - 'servicediscovery:ListServices'
+ - 'servicediscovery:UpdateService'
+ - 'servicediscovery:DeleteService'
+ Effect: Allow
+ Resource: '*'
+ CodeBuildRole:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ RoleName: !Sub 'CodeBuildRole-${CodeCommitRepositoryName}'
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service:
+ - codebuild.amazonaws.com
+ Action:
+ - 'sts:AssumeRole'
+ Path: /service-role/
+ Policies:
+ - PolicyName: !Sub 'CodeBuildNestedCFNAccessPolicy-${CodeCommitRepositoryName}'
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 'cloudformation:Get*'
+ - 'cloudformation:Describe*'
+ - 'cloudformation:List*'
+ Resource:
+ - '*'
+ - Effect: Allow
+ Action:
+ - 'codebuild:StartBuild'
+ Resource:
+ - 'Fn::Sub': >-
+ arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/*
+ - Effect: Allow
+ Action:
+ - 'codecommit:ListBranches'
+ - 'codecommit:ListRepositories'
+ - 'codecommit:BatchGetRepositories'
+ - 'codecommit:Get*'
+ - 'codecommit:GitPull'
+ Resource:
+ - !Sub >-
+ arn:aws:codecommit:${AWS::Region}:${AWS::AccountId}:${CodeCommitRepositoryName}
+ - Effect: Allow
+ Action:
+ - 'ec2:*'
+ - 'cloudformation:ValidateTemplate'
+ - 'elasticloadbalancing:Describe*'
+ - 'autoscaling:Describe*'
+ - 'iam:Get*'
+ - 'iam:List*'
+ - 'logs:Describe*'
+ - 'logs:Get*'
+ - 'tag:Get*'
+ - "ecr:*"
+ - "codedeploy:*"
+ - "ecs:*"
+ Resource:
+ - '*'
+ - Effect: Allow
+ Action:
+ - 'logs:CreateLogGroup'
+ - 'logs:CreateLogStream'
+ - 'logs:PutLogEvents'
+ Resource:
+ - 'Fn::Sub': >-
+ arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*
+ - Effect: Allow
+ Action:
+ - 's3:*'
+ Resource: '*'
+ - Effect: Allow
+ Action:
+ - 'lambda:UpdateFunctionCode'
+ - 'lambda:UpdateFunctionConfiguration'
+ - 'lambda:PublishLayerVersion'
+ - 'lambda:GetLayerVersion'
+ Resource: '*'
+ - Effect: Allow
+ Action:
+ - 'apigateway:GET'
+ - 'apigateway:POST'
+ Resource: '*'
+ PipelineRole:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ AssumeRolePolicyDocument:
+ Statement:
+ - Action:
+ - 'sts:AssumeRole'
+ Effect: Allow
+ Principal:
+ Service:
+ - codepipeline.amazonaws.com
+ Version: 2012-10-17
+ Path: /
+ Policies:
+ - PolicyName: !Sub 'CodePipelineAccess-${CodeCommitRepositoryName}'
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Action:
+ - 's3:*'
+ - 'cloudformation:CreateStack'
+ - 'cloudformation:DescribeStacks'
+ - 'cloudformation:DeleteStack'
+ - 'cloudformation:UpdateStack'
+ - 'cloudformation:CreateChangeSet'
+ - 'cloudformation:ExecuteChangeSet'
+ - 'cloudformation:DeleteChangeSet'
+ - 'cloudformation:DescribeChangeSet'
+ - 'cloudformation:SetStackPolicy'
+ - 'cloudformation:ValidateTemplate'
+ - 'iam:PassRole'
+ - 'sns:Publish'
+ - 'lambda:ListFunctions'
+ - 'lambda:InvokeFunction'
+ - 'ec2:Describe*'
+ - 'ec2:Get*'
+ - 'ec2:Search*'
+ - 'ec2:*Vpc*'
+ - 'ec2:*Gateway'
+ - 'ec2:*Tags'
+ - 'ec2:*Subnet*'
+ - 'ec2:*Route*'
+ - 'ec2:*SecurityGroup'
+ - 'ec2:allocate*'
+ - 'ec2:release*'
+ Effect: Allow
+ Resource: '*'
+ - Action:
+ - 'codecommit:GetUploadArchiveStatus'
+ - 'codecommit:CancelUploadArchive'
+ - 'codecommit:GetBranch'
+ - 'codecommit:GetCommit'
+ - 'codecommit:GetUploadStatus'
+ - 'codecommit:UploadArchive'
+ Effect: Allow
+ Resource: '*'
+ - Effect: Allow
+ Action:
+ - 'codebuild:*'
+ Resource:
+ - 'Fn::Sub': >-
+ arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/*
+
+Outputs:
+ CodeBuildRoleArn:
+ Description: The ARN of the role used by the CodeBuild projects.
+ Value: !GetAtt CodeBuildRole.Arn
+ Export:
+ Name: !Sub ${Environment}-${CodeCommitRepositoryName}-CodeBuildRoleArn
+
+ CFNRoleArn:
+ Description: The ARN of the role used by CloudFormation templates run from the automated pipeline.
+ Value: !GetAtt CFNRole.Arn
+ Export:
+ Name: !Sub ${Environment}-${CodeCommitRepositoryName}-CFNRoleArn
+
+ PipelineRoleArn:
+ Description: The ARN of the role used by the application's CodePipeline.
+ Value: !GetAtt PipelineRole.Arn
+ Export:
+ Name: !Sub ${Environment}-${CodeCommitRepositoryName}-PipelineRoleArn
diff --git a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
new file mode 100644
index 0000000000000000000000000000000000000000..17dd9c69dfcd27cc4c4ab2249d49028251770815
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
@@ -0,0 +1,550 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Creates all AWS resources used by OSDU's Indexer Service. Requires having previously setup the CodeCommit repository, as well as the CodePipeline (manual template).
+Parameters:
+ Environment:
+ Description: The name of the environment.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Environment can only be "dev/uat/prod".
+ Default: dev
+
+ DeploymentRegion:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ChildTemplateBasePath:
+ Description: >-
+ The base path for where child CloudFormation templates are located – can be relative or absolute, e.g.
+ https://s3.amazonaws.com/dev-osdu-cloudformation-scripts/Automated/
+ Type: String
+ AllowedPattern: '^https:\/\/s3.amazonaws.com\/.*\/$'
+ Default: https://s3.amazonaws.com/dev-osdu-cloudformation-scripts/Automated/
+
+ ApplicationName:
+ Description: >
+ The name of the application, should be equal to the repository name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-indexer
+
+ StorageApplicationName:
+ Description: >
+ The name of the storage application
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-storage
+
+ KeyName:
+ Description: >
+ Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
+ be created through CloudFormation, but instead must be uploaded through the AWS Console.
+ Type: AWS::EC2::KeyPair::KeyName
+ Default: ecs_indexer_key
+
+ DesiredCapacity:
+ Description: The default number of instances to launch in the ECS cluster.
+ Type: Number
+ Default: '1'
+
+ MinSize:
+ Description: Maximum number of instances that can be launched in the ECS cluster.
+ Type: Number
+ Default: '0'
+
+ MaxSize:
+ Description: Maximum number of instances that can be launched in the ECS cluster.
+ Type: Number
+ Default: '1'
+
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ Default: t3.large
+ AllowedValues:
+ - m5.large
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ - m5.12xlarge
+ - m5.16xlarge
+ - m5.24xlarge
+ - m4.large
+ - m4.xlarge
+ - m4.2xlarge
+ - m4.4xlarge
+ - m4.10xlarge
+ - m4.16xlarge
+ - t3.nano
+ - t3.micro
+ - t3.small
+ - t3.medium
+ - t3.large
+ - t3.xlarge
+ - t3.2xlarge
+ - c5.large
+ - c5.xlarge
+ - c5.2xlarge
+ - c5.4xlarge
+ - c5.12xlarge
+ - c5.16xlarge
+ - c5.24xlarge
+ - r5.large
+ - r5.xlarge
+ - r5.2xlarge
+ - r5.4xlarge
+ - r5.12xlarge
+ - r5.24xlarge
+ - r4.large
+ - r4.xlarge
+ - r4.2xlarge
+ - r4.4xlarge
+ - r4.8xlarge
+ - r4.16xlarge
+ - i3.large
+ - i3.xlarge
+ - i3.2xlarge
+ - i3.4xlarge
+ - i3.10xlarge
+ - i3.16xlarge
+ - x1e.xlarge
+ - x1e.2xlarge
+ - x1e.4xlarge
+ - x1e.8xlarge
+ - x1e.16xlarge
+ - x1e.32xlarge
+ ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
+
+ IndexerServiceIamUsername:
+ Description: The username of the service user for the OS Indexer Service.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Type: String
+ Default: service-user-os-indexer
+ MinLength: '1'
+ MaxLength: '64'
+
+ IndexerServiceIamKeyRotationSerial:
+ Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
+ Type: Number
+ Default: 1
+
+ SNSTopicName:
+ Description: >-
+ The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
+ Will be prefixed with the environment name.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer-messages
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+
+ SQSQueueName:
+ Description: >-
+ The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
+ Will be prefixed with the environment name.
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer-queue
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+
+ IndexCacheName:
+ Description: The name of the cache cluster for the legal tag cache. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerIndexCache
+
+ IndexCacheEngine:
+ Description: Which caching platform to use for the legal tag cache. Can be set to 'redis' or 'memcached'.
+ Type: String
+ AllowedValues:
+ - redis
+ - memcached
+ ConstraintDescription: Can only be "redis" or "memcached"
+ Default: redis
+
+ IndexCacheNodeInstanceType:
+ Description: The instance type for redis cache nodes for the legal tag cache.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: cache.t2.micro
+ AllowedValues:
+ - cache.m5.large
+ - cache.m5.xlarge
+ - cache.m5.2xlarge
+ - cache.m5.4xlarge
+ - cache.m5.12xlarge
+ - cache.m5.24xlarge
+ - cache.m4.large
+ - cache.m4.xlarge
+ - cache.m4.2xlarge
+ - cache.m4.4xlarge
+ - cache.m4.10xlarge
+ - cache.t2.micro
+ - cache.t2.small
+ - cache.t2.medium
+ - cache.c1.xlarge
+ - cache.r5.large
+ - cache.r5.xlarge
+ - cache.r5.2xlarge
+ - cache.r5.4xlarge
+ - cache.r5.12xlarge
+ - cache.r5.24xlarge
+ - cache.r4.large
+ - cache.r4.xlarge
+ - cache.r4.2xlarge
+ - cache.r4.4xlarge
+ - cache.r4.8xlarge
+ - cache.r4.16xlarge
+ Type: String
+
+ IndexCacheNumberOfCacheNodes:
+ Description: An integer value specifying the number of node in the redis cache for the legal tag cache.
+ Type: Number
+ Default: 1
+ MinValue: 1
+ MaxValue: 128
+
+ SchemaCacheName:
+ Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: indexerSchemaCache
+
+ SchemaCacheEngine:
+ Description: Which caching platform to use for the schema cache. Can be set to 'redis' or 'memcached'.
+ Type: String
+ AllowedValues:
+ - redis
+ - memcached
+ ConstraintDescription: Can only be "redis" or "memcached"
+ Default: redis
+
+ SchemaCacheNodeInstanceType:
+ Description: The instance type for redis cache nodes for the schema cache.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: cache.t2.micro
+ AllowedValues:
+ - cache.m5.large
+ - cache.m5.xlarge
+ - cache.m5.2xlarge
+ - cache.m5.4xlarge
+ - cache.m5.12xlarge
+ - cache.m5.24xlarge
+ - cache.m4.large
+ - cache.m4.xlarge
+ - cache.m4.2xlarge
+ - cache.m4.4xlarge
+ - cache.m4.10xlarge
+ - cache.t2.micro
+ - cache.t2.small
+ - cache.t2.medium
+ - cache.c1.xlarge
+ - cache.r5.large
+ - cache.r5.xlarge
+ - cache.r5.2xlarge
+ - cache.r5.4xlarge
+ - cache.r5.12xlarge
+ - cache.r5.24xlarge
+ - cache.r4.large
+ - cache.r4.xlarge
+ - cache.r4.2xlarge
+ - cache.r4.4xlarge
+ - cache.r4.8xlarge
+ - cache.r4.16xlarge
+ Type: String
+
+ SchemaCacheNumberOfCacheNodes:
+ Description: An integer value specifying the number of node in the redis cache for the schema cache.
+ Type: Number
+ Default: 1
+ MinValue: 1
+ MaxValue: 128
+
+ ElasticsearchDomainName:
+ Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer
+
+ ElasticsearchNodeInstanceType:
+ Description: The instance type for the main Elasticsearch nodes.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ DedicatedMasterInstanceType:
+ Description: >
+ The instance type for the dedicated master nodes. These nodes perform cluster management
+ tasks, but doesn't hold data or respond to data upload requests.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ NumberOfElasticsearchNodes:
+ Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+ Type: Number
+ Default: 1
+ MinValue: 1
+ MaxValue: 40
+
+ NumberOfDedicatedMasterNodes:
+ Description: An integer value specifying the number of dedicated master nodes.
+ Type: Number
+ Default: 2
+ MinValue: 2
+ MaxValue: 5
+
+ ZoneAwarenessEnabled:
+ Description: >
+ When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+ index shards that belong to a cluster across multiple AZs in the deployment region.
+ Type: String
+ AllowedValues:
+ - true
+ - false
+ Default: false
+
+ ElasticsearchVersion:
+ Description: >
+ The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+ that an update requires a full replacement of the Elasticsearch cluster.
+ Type: String
+ AllowedValues:
+ - 1.5
+ - 2.3
+ - 5.1
+ - 5.3
+ - 5.5
+ - 5.6
+ - 6.0
+ - 6.2
+ - 6.3
+ - 6.4
+ - 6.5
+ - 6.6
+ - 6.8
+ - 6.8
+ - 7.1
+ Default: 6.8
+
+ EBSVolumeSize:
+ Description: >
+ The size of the EBS volume, in GiB, (per instance; total cluster size =
+ EBS volume size x Instance count). Maximum size varies by instance type, from 35GiB
+ for t2 instances, up to 12TiB for r5.12xlarge.
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 12000
+
+ ECSPort:
+ Description: The port that the ECS Service will listen on.
+ Type: Number
+ Default: 80
+ MinValue: 1
+ MaxValue: 65535
+
+ ECSCPUAllocation:
+ Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
+ Type: Number
+ Default: 1024
+ MinValue: 10
+ MaxValue: 65535
+
+ ECSMemoryAllocation:
+ Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
+ Type: Number
+ Default: 2048
+ MinValue: 256
+ MaxValue: 131072
+
+Resources:
+
+ #### Shared Resources ################################################################
+
+ IAMCredentialsStack:
+ Type: 'AWS::CloudFormation::Stack'
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, iam-credentials.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ IndexerServiceIamUsername: !Ref IndexerServiceIamUsername
+ IndexerServiceIamKeyRotationSerial: !Ref IndexerServiceIamKeyRotationSerial
+
+ MessageBusSNSStack:
+ Type: 'AWS::CloudFormation::Stack'
+ Properties:
+ TemplateURL: !Join ['', [!Ref ChildTemplateBasePath, sns-topic.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ SNSTopicName: !Ref SNSTopicName
+ SQSQueueName: !Ref SQSQueueName
+
+ #### ECS Resources ###################################################################
+
+ ECSNetworkStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: IAMCredentialsStack
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, ecs-network.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ApplicationName: !Ref ApplicationName
+ ECSPort: !Ref ECSPort
+
+ ECSClusterStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: [SchemaCacheStack, IndexCacheStack]
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, ecs-cluster.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ApplicationName: !Ref ApplicationName
+ KeyName: !Ref KeyName
+ DesiredCapacity: !Ref DesiredCapacity
+ MaxSize: !Ref MaxSize
+ InstanceType: !Ref InstanceType
+ SchemaCacheName: !Ref SchemaCacheName
+ IndexCacheName: !Ref IndexCacheName
+ ECSPort: !Ref ECSPort
+ SNSTopicName: !Ref SNSTopicName
+ ECSMemoryAllocation: !Ref ECSMemoryAllocation
+
+ #### Caching Resources ###############################################################
+
+ IndexCacheStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: ECSNetworkStack
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, cache.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ApplicationName: !Ref ApplicationName
+ CacheName: !Ref IndexCacheName
+ CacheEngine: !Ref IndexCacheEngine
+ NodeInstanceType: !Ref IndexCacheNodeInstanceType
+ NumberOfCacheNodes: !Ref IndexCacheNumberOfCacheNodes
+
+ SchemaCacheStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: ECSNetworkStack
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, cache.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ApplicationName: !Ref ApplicationName
+ CacheName: !Ref SchemaCacheName
+ CacheEngine: !Ref SchemaCacheEngine
+ NodeInstanceType: !Ref SchemaCacheNodeInstanceType
+ NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
+
+ #### Elasticsearch Resources #########################################################
+ # TODO: Add VPCs to Elasticsearch
+
+ ElasticsearchStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: IAMCredentialsStack
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, elasticsearch.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ElasticsearchDomainName: !Ref ElasticsearchDomainName
+ ElasticsearchNodeInstanceType: !Ref ElasticsearchNodeInstanceType
+ DedicatedMasterInstanceType: !Ref DedicatedMasterInstanceType
+ NumberOfElasticsearchNodes: !Ref NumberOfElasticsearchNodes
+ NumberOfDedicatedMasterNodes: !Ref NumberOfDedicatedMasterNodes
+ ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+ ElasticsearchVersion: !Ref ElasticsearchVersion
+ EBSVolumeSize: !Ref EBSVolumeSize
diff --git a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
new file mode 100644
index 0000000000000000000000000000000000000000..c24b72c65cdf1c366e0c85096177c12f30107a3c
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
@@ -0,0 +1,39 @@
+{
+ "Parameters" : {
+ "Environment" : "dev",
+ "DeploymentRegion" : "us-east-1",
+ "ChildTemplateBasePath" : "https://s3.amazonaws.com/dev-os-indexer-cloudformation-scripts/Automated/",
+ "ApplicationName" : "os-indexer",
+ "KeyName": "indexer-ecs-keypair",
+ "DesiredCapacity": "2",
+ "MinSize": "0",
+ "MaxSize": "3",
+ "InstanceType": "t3.large",
+ "IndexerServiceIamUsername": "service-user-os-indexer",
+ "IndexerServiceIamKeyRotationSerial": "1",
+ "SNSTopicName": "osdu-indexer-messages",
+ "SQSQueueName": "osdu-indexer-queue",
+ "IndexCacheName": "indexerIndexCache",
+ "IndexCacheEngine": "redis",
+ "IndexCacheNodeInstanceType": "cache.t2.micro",
+ "IndexCacheNumberOfCacheNodes": "1",
+ "SchemaCacheName": "indexerSchemaCache",
+ "SchemaCacheEngine": "redis",
+ "SchemaCacheNodeInstanceType": "cache.t2.micro",
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10",
+ "ECSPort": "80",
+ "ECSCPUAllocation": "1024",
+ "ECSMemoryAllocation": "3072"
+ },
+ "Tags" : {
+ "Environment" : "dev"
+ }
+}
diff --git a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
new file mode 100644
index 0000000000000000000000000000000000000000..013cc5a2bd6a96f28185a4b08291e228f18f9948
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
@@ -0,0 +1,39 @@
+{
+ "Parameters" : {
+ "Environment" : "prod",
+ "DeploymentRegion" : "us-east-1",
+ "ChildTemplateBasePath" : "https://s3.amazonaws.com/prod-os-indexer-cloudformation-scripts/Automated/",
+ "ApplicationName" : "os-indexer",
+ "KeyName": "indexer-ecs-keypair",
+ "DesiredCapacity": "2",
+ "MinSize": "0",
+ "MaxSize": "3",
+ "InstanceType": "t3.large",
+ "IndexerServiceIamUsername": "service-user-os-indexer",
+ "IndexerServiceIamKeyRotationSerial": "1",
+ "SNSTopicName": "osdu-indexer-messages",
+ "SQSQueueName": "osdu-indexer-queue",
+ "IndexCacheName": "indexerIndexCache",
+ "IndexCacheEngine": "redis",
+ "IndexCacheNodeInstanceType": "cache.t2.micro",
+ "IndexCacheNumberOfCacheNodes": "1",
+ "SchemaCacheName": "indexerSchemaCache",
+ "SchemaCacheEngine": "redis",
+ "SchemaCacheNodeInstanceType": "cache.t2.micro",
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10",
+ "ECSPort": "80",
+ "ECSCPUAllocation": "1024",
+ "ECSMemoryAllocation": "3072"
+ },
+ "Tags" : {
+ "Environment" : "prod"
+ }
+}
diff --git a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
new file mode 100644
index 0000000000000000000000000000000000000000..c5e5a3af82545b5617f743aae1485117413132d2
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
@@ -0,0 +1,39 @@
+{
+ "Parameters" : {
+ "Environment" : "uat",
+ "DeploymentRegion" : "us-east-1",
+ "ChildTemplateBasePath" : "https://s3.amazonaws.com/uat-os-indexer-cloudformation-scripts/Automated/",
+ "ApplicationName" : "os-indexer",
+ "KeyName": "indexer-ecs-keypair",
+ "DesiredCapacity": "2",
+ "MinSize": "0",
+ "MaxSize": "3",
+ "InstanceType": "t3.large",
+ "IndexerServiceIamUsername": "service-user-os-indexer",
+ "IndexerServiceIamKeyRotationSerial": "1",
+ "SNSTopicName": "osdu-indexer-messages",
+ "SQSQueueName": "osdu-indexer-queue",
+ "IndexCacheName": "indexerIndexCache",
+ "IndexCacheEngine": "redis",
+ "IndexCacheNodeInstanceType": "cache.t2.micro",
+ "IndexCacheNumberOfCacheNodes": "1",
+ "SchemaCacheName": "indexerSchemaCache",
+ "SchemaCacheEngine": "redis",
+ "SchemaCacheNodeInstanceType": "cache.t2.micro",
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10",
+ "ECSPort": "80",
+ "ECSCPUAllocation": "1024",
+ "ECSMemoryAllocation": "3072"
+ },
+ "Tags" : {
+ "Environment" : "uat"
+ }
+}
diff --git a/provider/indexer-aws/Dockerfile b/provider/indexer-aws/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..eee4c09535e93300729593d5d7e30a7ee33c3f5a
--- /dev/null
+++ b/provider/indexer-aws/Dockerfile
@@ -0,0 +1,7 @@
+FROM amazoncorretto:8
+
+WORKDIR /
+COPY provider/indexer-aws/target/indexer-aws-0.0.1-SNAPSHOT-spring-boot.jar indexer-aws-0.0.1-SNAPSHOT-spring-boot.jar
+EXPOSE 8080
+
+CMD ["java","-jar", "indexer-aws-0.0.1-SNAPSHOT-spring-boot.jar"]
\ No newline at end of file
diff --git a/provider/indexer-aws/azure-build.yml b/provider/indexer-aws/azure-build.yml
new file mode 100644
index 0000000000000000000000000000000000000000..78e2e712788ead2ab0269e0c7a5b7b2be0d1ea1b
--- /dev/null
+++ b/provider/indexer-aws/azure-build.yml
@@ -0,0 +1,50 @@
+# Maven
+# Build your Java project and run tests with Apache Maven.
+# Add steps that analyze code, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/java
+trigger:
+ branches:
+ include:
+ - master
+ paths:
+ exclude:
+ - README.md
+ - .gitignore
+pool:
+ name: dps-build
+ demands: maven
+steps:
+ - task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'install'
+ - task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./provider/indexer-aws/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U -P indexer-aws'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'install'
+ - task: CopyFiles@2
+ displayName: 'Copy AWS artifacts for maven deploy to: $(build.artifactstagingdirectory)'
+ inputs:
+ SourceFolder:
+ Contents: |
+ provider/indexer-aws/CloudFormation
+ provider/indexer-aws/buildspec-post-deploy.yml
+ provider/indexer-aws/buildspec-pre-deploy.yml
+ provider/indexer-aws/target/*-spring-boot.jar
+ TargetFolder: '$(build.artifactstagingdirectory)'
+
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifact: drop'
+ inputs:
+ PathtoPublish: '$(build.artifactstagingdirectory)'
+ ArtifactName: 'drop'
+ publishLocation: 'Container'
+ condition: succeededOrFailed()
diff --git a/provider/indexer-aws/buildspec-post-deploy.yml b/provider/indexer-aws/buildspec-post-deploy.yml
new file mode 100644
index 0000000000000000000000000000000000000000..02fd6e3a518ef177e7140ac5a164f94a60d26c46
--- /dev/null
+++ b/provider/indexer-aws/buildspec-post-deploy.yml
@@ -0,0 +1,88 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 0.2
+
+phases:
+ install:
+ runtime-versions:
+ java: openjdk8
+ commands:
+ - echo Entered the install phase...
+ - apt-get update -y
+ - apt-get install -y maven
+ - java -version
+ - mvn clean # .m2 is not created until the first Maven command
+ - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
+ - cat /root/.m2/settings.xml
+ - java -version
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - echo $JAVA_HOME
+ - mvn -version
+ - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
+ - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
+ pre_build:
+ commands:
+ - echo Logging in to Amazon ECR...
+ - $(aws ecr get-login --no-include-email --region $AWS_REGION)
+ - echo $AWS_ACCOUNT_ID
+ - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$IMAGE_REPO_NAME # build and store the ECR repo URI
+ - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}') # generate a version tag from the commit hash for the Docker image
+ - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) # get the commit hash
+ build:
+ commands:
+ - echo Indexer-core Java build started on `date`...
+ - echo os-indexer Java build started on `date`...
+ - java -version
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - mvn -version
+ - echo Look below for repository bucket
+ - echo $S3_MAVEN_REPOSITORY
+ - echo Setting environment variables from CloudFormation Exports... # use the AWS CLI commands to query for the CloudFormation export values created in the previous step and set the required environment variables
+ - echo Environment - $ENVIRONMENT
+ - echo SchemaCacheName - $SCHEMA_CACHE_NAME
+ - echo IndexCacheName - $INDEX_CACHE_NAME
+ - echo AWSRegion - $AWS_REGION
+ - export CACHE_CLUSTER_SCHEMA_ENDPOINT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$SCHEMA_CACHE_NAME-RedisEndpointAddress'].[Value]" --output text --region $AWS_REGION)
+ - export CACHE_CLUSTER_SCHEMA_PORT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$SCHEMA_CACHE_NAME-RedisEndpointPort'].[Value]" --output text --region $AWS_REGION)
+ - export CACHE_CLUSTER_INDEX_ENDPOINT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$INDEX_CACHE_NAME-RedisEndpointAddress'].[Value]" --output text --region $AWS_REGION)
+ - export CACHE_CLUSTER_INDEX_PORT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$INDEX_CACHE_NAME-RedisEndpointPort'].[Value]" --output text --region $AWS_REGION)
+ - export S3_DATA_BUCKET=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-S3BucketDataStorage'].[Value]" --output text --region $AWS_REGION)
+ - export SNS_TOPIC_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-OSDUIndexerSNSTopic'].[Value]" --output text --region $AWS_REGION)
+ - echo ...finished setting environment variables!
+ - echo All environment variables
+ - printenv
+ - mvn clean test -pl indexer-core,provider/indexer-aws -Ddeployment.environment=$ENVIRONMENT -Ddeployment.repositorybucket=$S3_MAVEN_REPOSITORY -Daws.accessKeyId=$AWS_ACCESS_KEY_ID -Daws.secretKey=$AWS_SECRET_KEY -Dazure.devops.token=$VSTS_FEED_TOKEN -DCACHE_CLUSTER_SCHEMA_ENDPOINT=$CACHE_CLUSTER_SCHEMA_ENDPOINT -DCACHE_CLUSTER_SCHEMA_PORT=$CACHE_CLUSTER_SCHEMA_PORT -DCACHE_CLUSTER_GROUP_ENDPOINT=$CACHE_CLUSTER_SCHEMA_ENDPOINT -DCACHE_CLUSTER_SCHEMA_PORT=$CACHE_CLUSTER_SCHEMA_PORT -DCACHE_CLUSTER_INDEX_ENDPOINT=$CACHE_CLUSTER_INDEX_ENDPOINT -DCACHE_CLUSTER_INDEX_PORT=$CACHE_CLUSTER_INDEX_PORT -DS3_DATA_BUCKET=$S3_DATA_BUCKET -DSNS_TOPIC_NAME=$SNS_TOPIC_NAME -DAWS_ACCOUNT_ID=$AWS_ACCOUNT_ID -DAWS_REGION=$AWS_REGION -DENVIRONMENT=$ENVIRONMENT
+ - echo ...os-indexer Java build completed on `date`.
+ - echo os-indexer beginning packaging to jar...
+ - mvn clean deploy -pl indexer-core,provider/indexer-aws -Ddeployment.environment=$ENVIRONMENT -Ddeployment.repositorybucket=$S3_MAVEN_REPOSITORY -Daws.accessKeyId=$AWS_ACCESS_KEY_ID_MAVEN -Daws.secretKey=$AWS_SECRET_ACCESS_KEY_MAVEN -Dazure.devops.token=$VSTS_FEED_TOKEN
+ - echo os-indexer Docker image build started on `date`...
+ - docker build -f provider/indexer-aws/Dockerfile -t $REPOSITORY_URI:latest .
+ - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG
+ - echo ...os-indexer Docker image build completed on `date`.
+ - echo Pushing the Docker image to ECR...
+ - docker push $REPOSITORY_URI:latest
+ - docker push $REPOSITORY_URI:$IMAGE_TAG
+ - echo Docker image pushed to ECR successfully!
+ - ECS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsClusterName'].[Value]" --output text --region $AWS_REGION)
+ - ECS_SERVICE_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsServiceName'].[Value]" --output text --region $AWS_REGION)
+ - aws ecs update-service --cluster $ECS_CLUSTER_NAME --service $ECS_SERVICE_NAME --force-new-deployment # force a new deployment with the updated image
+
+cache:
+ paths:
+ - '/root/.m2/**/*'
+
+artifacts:
+ files:
+ - '**/*'
diff --git a/provider/indexer-aws/buildspec-pre-deploy.yml b/provider/indexer-aws/buildspec-pre-deploy.yml
new file mode 100644
index 0000000000000000000000000000000000000000..baf44c6068f3ec612bf4965e95c8f094e9ec8aae
--- /dev/null
+++ b/provider/indexer-aws/buildspec-pre-deploy.yml
@@ -0,0 +1,69 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 0.2
+
+phases:
+# pre-build:
+# commands:
+# - echo Logging in to Amazon ECR...
+# - aws --version
+# - $(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)
+# - REPOSITORY_URI=012345678910.dkr.ecr.us-east-1.amazonaws.com/hello-world
+# - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
+# - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}')
+ install:
+ runtime-versions:
+ java: openjdk8
+ docker: 18
+ commands:
+ - echo Entered the install phase...
+ - apt-get update -y
+ - apt-get install -y maven
+ - java -version
+ - echo $JAVA_HOME
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - echo $JAVA_HOME
+ - mvn clean # .m2 is not created until the first Maven command
+ - cp ./indexer-core/maven/settings.xml /root/.m2/settings.xml # replace the default settings.xml with our custom one
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - mvn -version
+ build:
+ commands:
+ - echo Starting 'Copying CloudFormation scripts to S3://$CFN_S3_BUCKET'
+ - pwd
+ - ls
+ - aws s3 cp ./provider/indexer-aws/CloudFormation "s3://$CFN_S3_BUCKET" --exclude "*" --include "*.yml" --recursive --debug
+ - echo Ending 'Ending CloudFormation scripts to S3://$CFN_S3_BUCKET'
+# - echo os-indexer build started on `date`...
+ - pwd
+ - ls -R -la
+ - java -version
+ - echo $JAVA_HOME
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - mvn -version
+# - mvn test
+ post_build:
+ commands:
+# - echo ...os-indexer build completed on `date`
+# - echo os-indexer beginning packaging to jar...
+# - mvn package
+
+cache:
+ paths:
+ - '/root/.m2/**/*'
+
+artifacts:
+ files:
+ - '**/*'
diff --git a/provider/indexer-aws/maven/settings.xml b/provider/indexer-aws/maven/settings.xml
new file mode 100644
index 0000000000000000000000000000000000000000..54c4a7227d6191761d3b091148e6182d45d129cf
--- /dev/null
+++ b/provider/indexer-aws/maven/settings.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+
+ <servers>
+ <server>
+ <id>os-core</id>
+ <username>slb-des-ext-collaboration</username>
+ <password>${VSTS_FEED_TOKEN}</password>
+ </server>
+
+ </servers>
+</settings>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..872ab23987ffc5ce3fc4e3b2f5810899ac1ac504
--- /dev/null
+++ b/provider/indexer-aws/pom.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0"?>
+<!--
+ Copyright © Amazon Web Services
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0.2</version>
+ <relativePath>../..</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>indexer-aws</artifactId>
+ <description>Storage service on AWS</description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <aws.version>1.11.637</aws.version>
+ <deployment.environment>dev</deployment.environment>
+ <deployment.repositorybucket>osdu-local-maven-repository</deployment.repositorybucket>
+ </properties>
+
+ <dependencies>
+ <!-- Internal packages -->
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>0.0.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>1.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu.core.aws</groupId>
+ <artifactId>aws-osdu-util</artifactId>
+ <version>0.0.4</version>
+ </dependency>
+
+ <!-- AWS managed packages -->
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <version>1.11.651</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-request-signing-apache-interceptor</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.11.327</version>
+ </dependency>
+
+ <!-- Third party Apache 2.0 license packages -->
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.oauth</groupId>
+ <artifactId>spring-security-oauth2</artifactId>
+ <version>2.3.6.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-jwt</artifactId>
+ <version>1.0.10.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-jose</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>7.6.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
+ <version>1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!-- Testing packages -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.10.19</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.data</groupId>
+ <artifactId>spring-data-commons</artifactId>
+ <version>2.1.10.RELEASE</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>org.opengroup.osdu.indexer.aws.IndexerAwsApplication</mainClass>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <extensions>
+ <extension>
+ <groupId>com.github.seahen</groupId>
+ <artifactId>maven-s3-wagon</artifactId>
+ <version>1.3.0</version>
+ </extension>
+ </extensions>
+ </build>
+ <repositories>
+ <repository>
+ <id>os-core</id>
+ <url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+
+ <repository>
+ <id>local.release</id>
+ <url>file:../../local-release-dir</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+</project>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..501bdddb2cf77765a701fb3651cfff5e5340713c
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
@@ -0,0 +1,32 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+
+@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
+@ComponentScan({"org.opengroup.osdu.core.common", "org.opengroup.osdu.indexer"})
+public class IndexerAwsApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerAwsApplication.class, args);
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/ServiceLogId.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ElasticCredentialsCacheImpl.java
similarity index 60%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/ServiceLogId.java
rename to provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ElasticCredentialsCacheImpl.java
index a09c34132e165f2240e7faf2e3cdf2df3e2ee0c3..db363d92d87c633e3acc37278e4d1c69fdec1acf 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/ServiceLogId.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ElasticCredentialsCacheImpl.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,27 +12,31 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.logging;
+package org.opengroup.osdu.indexer.aws.cache;
-import org.opengroup.osdu.is.core.logging.LogId;
+import org.opengroup.osdu.core.common.spi.coreis.IElasticCredentialsCache;
import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
@Component
-@RequestScope
-public class ServiceLogId implements LogId {
+public class ElasticCredentialsCacheImpl implements IElasticCredentialsCache {
+
@Override
- public String getRequestLog() {
- return "indexer.request";
+ public void put(Object o, Object o2) {
+
}
@Override
- public String getAuditLog() {
- return "indexer.audit";
+ public Object get(Object o) {
+ return null;
}
@Override
- public String getAppLog() {
- return "indexer.app";
+ public void delete(Object o) {
+
+ }
+
+ @Override
+ public void clearAll() {
+
}
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..2e5976c76da15079fd330b53d038b2528fdfd85e
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
@@ -0,0 +1,59 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.RedisCache;
+import org.opengroup.osdu.core.common.spi.coreis.IIndexCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseable {
+
+ private RedisCache<String, Boolean> cache;
+
+ public IndexCacheImpl(@Value("${aws.elasticache.cluster.index.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.index.port}") final String REDIS_SEARCH_PORT,
+ @Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..9434ed3b67731f362a827631ac62794fcafe694c
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
@@ -0,0 +1,59 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.RedisCache;
+import org.opengroup.osdu.core.common.spi.indexer.ISchemaCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCacheImpl implements ISchemaCache<String, String>, AutoCloseable {
+
+ private RedisCache<String, String> cache;
+
+ public SchemaCacheImpl(@Value("${aws.elasticache.cluster.schema.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.schema.port}") final String REDIS_SEARCH_PORT,
+ @Value("${aws.elasticache.cluster.schema.expiration}") final String SCHEMA_CACHE_EXPIRATION) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexingStatus.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/DpsLogFactory.java
similarity index 50%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexingStatus.java
rename to provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/DpsLogFactory.java
index 5f86356775ed884a639a0f72d2d983c3ec2a78b3..b3bde6b2e7a4cc68f92fd2edfc0788acda9fea21 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexingStatus.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/DpsLogFactory.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,27 +12,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.indexer.aws.di;
-public enum IndexingStatus {
+import org.opengroup.osdu.core.common.model.core.DpsLog;
+import org.opengroup.osdu.core.common.service.core.Log;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
- PROCESSING(0),
+@Component
+public class DpsLogFactory extends AbstractFactoryBean<DpsLog> {
- SUCCESS(1),
-
- WARN(2),
-
- SKIP(3),
-
- FAIL(4);
-
- private final Integer severity;
-
- IndexingStatus(int severity) {
- this.severity = severity;
+ @Override
+ protected DpsLog createInstance() throws Exception {
+ return new Log();
}
- public boolean isWorseThan(IndexingStatus other) {
- return this.severity > other.severity;
+ @Override
+ public Class<?> getObjectType() {
+ return DpsLog.class;
}
}
\ No newline at end of file
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..92522a9e9a4de7a73bbf7347c53807bd08425f14
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
@@ -0,0 +1,36 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.di;
+
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Component
+@Primary
+public class EntitlementsFactoryImpl implements IEntitlementsFactory {
+ @Value("${aws.lambda.get-groups-function-name}")
+ private String getGroupsFunctionName;
+
+ @Override
+ public IEntitlementsService create(DpsHeaders headers) {
+ EntitlementsServiceImpl service = new EntitlementsServiceImpl(headers);
+ service.setEntitlementsServiceHelper(getGroupsFunctionName);
+ return service;
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..e4cb22e094c14a3d3aca41a9853d01f2721a831c
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
@@ -0,0 +1,139 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.di;
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.lambda.invoke.LambdaFunctionException;
+import com.amazonaws.services.lambda.invoke.LambdaSerializationException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.entitlements.EntitlementsException;
+import org.opengroup.osdu.core.common.model.core.entitlements.MemberInfo;
+import org.opengroup.osdu.core.common.model.core.entitlements.Members;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+import org.opengroup.osdu.core.common.model.core.entitlements.*;
+import org.opengroup.osdu.core.aws.entitlements.*;
+import org.opengroup.osdu.core.common.service.core.HttpResponse;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.http.HttpStatus;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EntitlementsServiceImpl implements IEntitlementsService {
+ private DpsHeaders dpsHeaders;
+ private EntitlementsServiceHelper entitlementsServiceHelper;
+
+ private final static String ACCESS_DENIED = "Access denied";
+ private final static String ACCESS_DENIED_MSG = "The user is not authorized to perform this action";
+
+ @Inject
+ @Lazy
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ public EntitlementsServiceImpl(DpsHeaders headers){
+ this.dpsHeaders = headers;
+ }
+
+ public void setEntitlementsServiceHelper(String getGroupsFunctionName){
+ entitlementsServiceHelper = new EntitlementsServiceHelper(Regions.US_EAST_1, getGroupsFunctionName);
+ }
+
+ @Override
+ public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Groups getGroups() throws EntitlementsException {
+ Groups groups;
+ GroupsRequest request = entitlementsServiceHelper.constructRequest(this.dpsHeaders.getHeaders());
+
+ try{
+ GroupsResult groupsResult = entitlementsServiceHelper.getGroups(request);
+ groups = getGroupsFromResult(groupsResult);
+ } catch (JsonProcessingException e) {
+ throw new EntitlementsException(e.getMessage(), new HttpResponse());
+ } catch (LambdaFunctionException e){
+ throw new EntitlementsException(e.getMessage(), new HttpResponse());
+ } catch (LambdaSerializationException e){
+ throw new EntitlementsException(e.getMessage(), new HttpResponse());
+ } catch (IOException e){
+ throw new EntitlementsException(e.getMessage(), new HttpResponse());
+ }
+
+ return groups;
+ }
+
+ @Override
+ public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void deleteMember(String s, String s1) throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Groups authorizeAny(String... strings) throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void authenticate() throws EntitlementsException {
+ throw new NotImplementedException();
+ }
+
+ private Groups getGroupsFromResult(GroupsResult result) throws EntitlementsException, IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ Groups groups = new Groups();
+ if(result.statusCode == HttpStatus.OK.value()) {
+ TypeReference<List<GroupInfoRaw>> mapType = new TypeReference<List<GroupInfoRaw>>() {};
+ List<GroupInfoRaw> groupInfosRaw = mapper.readValue(result.body, mapType);
+ List<GroupInfo> groupInfos = new ArrayList<>();
+ for(GroupInfoRaw groupInfoRaw : groupInfosRaw){
+ GroupInfo groupInfo = new GroupInfo();
+ groupInfo.setDescription(groupInfoRaw.groupDescription);
+ groupInfo.setEmail(groupInfoRaw.groupEmail);
+ groupInfo.setName(groupInfoRaw.groupName);
+ groupInfos.add(groupInfo);
+ }
+ groups.setDesId(result.headers.get(RequestKeys.USER_HEADER_KEY));
+ groups.setMemberEmail(result.headers.get(RequestKeys.USER_HEADER_KEY));
+ groups.setGroups(groupInfos);
+ } else {
+ if(result.statusCode == HttpStatus.UNAUTHORIZED.value()){
+ throw new AppException(HttpStatus.FORBIDDEN.value(), ACCESS_DENIED, ACCESS_DENIED_MSG);
+ } else {
+ throw new EntitlementsException(String.format("Getting groups for user returned %s status code",
+ result.statusCode), new HttpResponse());
+ }
+ }
+ return groups;
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantFactoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..15f07588070366c556c574b590c18db4e516e59e
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantFactoryImpl.java
@@ -0,0 +1,63 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.di;
+
+import org.opengroup.osdu.core.common.model.core.ICache;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.springframework.stereotype.Component;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+ public static final String[] REGISTERED_TENANTS = new String[] {"common", "opendes"};
+ private Map<String, TenantInfo> tenants;
+
+ public TenantFactoryImpl()
+ {
+ this.tenants = new HashMap<>();
+ for (String tenantName : REGISTERED_TENANTS) {
+ TenantInfo ti = new TenantInfo();
+ ti.setName(tenantName);
+ this.tenants.put(tenantName, ti);
+ }
+ }
+ @Override
+ public boolean exists(String tenantName) {
+ return this.tenants.containsKey(tenantName);
+ }
+
+ @Override
+ public TenantInfo getTenantInfo(String tenantName) {
+ return this.tenants.get(tenantName); }
+
+ @Override
+ public Collection<TenantInfo> listTenantInfo() {
+ return this.tenants.values();
+ }
+
+ @Override
+ public <V> ICache<String, V> createCache(String s, String s1, int i, int i1, Class<V> aClass) {
+ return null;
+ }
+
+ @Override
+ public void flushCache() {
+
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoFactory.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..f6767fa96d53daf18e6268763444348baa4c4182
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/TenantInfoFactory.java
@@ -0,0 +1,45 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.di;
+
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Log
+@Component
+@RequestScope
+public class TenantInfoFactory extends AbstractFactoryBean<TenantInfo> {
+ @Autowired
+ private ITenantFactory tenantFactory;
+ @Autowired
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders headers;
+ @Override
+ protected TenantInfo createInstance() throws Exception {
+ String id = this.headers.getPartitionIdWithFallbackToAccountId();
+ return this.tenantFactory.getTenantInfo(id);
+ }
+ @Override
+ public Class<?> getObjectType() {
+ return TenantInfo.class;
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..97f727d2e1995ee558a66a807767ac0d4166f4c9
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -0,0 +1,39 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.persistence;
+
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.spi.coreis.ElasticRepository;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticRepositoryImpl implements ElasticRepository {
+
+ // TODO: Will need to be implemented later
+
+ @Value("${aws.es.host}")
+ String host;
+
+ int port = 8080;
+
+ String userNameAndPassword = "testing";
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+ return new ClusterSettings(host, port, userNameAndPassword);
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..273fb5d19a1c0988f891bafd2bd92671095e3492
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -0,0 +1,69 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.publish;
+
+import com.amazonaws.services.sns.model.MessageAttributeValue;
+import com.amazonaws.services.sns.model.PublishRequest;
+import com.amazonaws.services.sns.AmazonSNS;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
+import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
+import org.opengroup.osdu.core.common.spi.indexer.IPublisher;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class PublisherImpl implements IPublisher {
+
+ AmazonSNS snsClient;
+
+ @Value("${aws.sns.arn}")
+ private String amazonSNSTopic;
+
+ @Value("${aws.sns.region}")
+ private String amazonSNSRegion;
+
+ @Inject
+ public void init(){
+ AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
+ snsClient = snsConfig.AmazonSNS();
+ }
+
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
+ {
+ // Attributes
+ Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ headers.addCorrelationIdIfMissing();
+ messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getCorrelationId()));
+
+ PublishRequest publishRequest = new PublishRequestBuilder().generatePublishRequest("data", indexerBatchStatus.getStatusesList(), messageAttributes, amazonSNSTopic);
+
+ snsClient.publish(publishRequest);
+ }
+
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/security/BasicAuthSecurityConfig.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/security/BasicAuthSecurityConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..6585ebc959b9fb878de01658d042c53bae1271e6
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/security/BasicAuthSecurityConfig.java
@@ -0,0 +1,30 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.security;
+
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http.httpBasic().disable()
+ .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
new file mode 100644
index 0000000000000000000000000000000000000000..c84bfe61798c7be957769f4a347040fb0eedc0ab
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
@@ -0,0 +1,64 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.service;
+
+import com.amazonaws.auth.AWS4Signer;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequestInterceptor;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.aws.iam.IAMConfig;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+// TODO: Elastic Client Handler should be designed to allow cloud providers to implement their own handler if not we have to inherited
+// SPI needs to be refactored
+@Primary
+@Component
+public class ElasticClientHandlerAws extends ElasticClientHandler {
+
+ @Value("${aws.es.serviceName}")
+ private String serviceName;
+
+ @Value("${aws.region}")
+ private String region;
+
+ @Value("${aws.es.host}")
+ String host;
+
+ public ElasticClientHandlerAws() {
+ }
+
+ @Override
+ public RestHighLevelClient createRestClient() {
+
+ return esClient();
+ }
+
+ private RestHighLevelClient esClient() {
+ AWSCredentialsProvider credentials = new IAMConfig().amazonAWSCredentials();
+ AWS4Signer signer = new AWS4Signer();
+ signer.setServiceName(serviceName);
+ signer.setRegionName(region);
+ HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentials);
+ return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor)));
+
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ConversionStatus.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/ComponentConfiguration.java
similarity index 62%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ConversionStatus.java
rename to provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/ComponentConfiguration.java
index 4160ee5bc4a928ea4aaf6bf3ed8d7d0af0c1f9f4..5781cd33033f01741e77cd667eacd2f36a025af8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ConversionStatus.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/ComponentConfiguration.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,17 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.indexer.aws.util;
-import lombok.Builder;
-import lombok.Data;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.stereotype.Component;
-import java.util.List;
-
-@Data
-@Builder
-public class ConversionStatus {
- private String id;
- private String status;
- private List<String> errors;
+@Component
+@ComponentScan({"org.opengroup.osdu.core.common", "com.amazonaws.osdu.util", "org.opengroup.osdu.is"})
+public class ComponentConfiguration {
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ee989889f040a03d9371e87597c46429b88e1d9
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java
@@ -0,0 +1,124 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.util;
+
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.SlbHeaders;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.service.coreis.Preconditions;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Primary
+@Log
+@Component
+public class HeadersInfoAwsImpl implements IHeadersInfo {
+
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders headersMap;
+
+
+ private static final HashSet<String> FORBIDDEN_FROM_LOGGING = new HashSet<>();
+ static {
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.AUTHORIZATION);
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.ON_BEHALF_OF);
+ }
+
+ /**
+ * Get list of current headers
+ * @return DpsHeaders headers
+ */
+ @Override
+ public DpsHeaders getHeaders() {
+ if (headersMap == null) {
+ log.warning("Headers Map DpsHeaders is null");
+ // throw to prevent null reference exception below
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid Headers", "Headers Map DpsHeaders is null");
+ }
+ DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders());
+ return headers;
+ }
+
+ /**
+ * get current logged in user
+ * @return userEmail
+ */
+ @Override
+ public String getUser() {
+ // TODO: This class is going to be deprecated soon so this whole class will go away. Additionally there shouldn't be a user email attached to the request because this is a message from storage.
+ return "user";
+ }
+
+ /**
+ * get partition id and fallback to account id
+ * @return Partition ID
+ */
+ @Override
+ public String getPartitionId() {
+ return getHeaders().getPartitionIdWithFallbackToAccountId();
+ }
+
+ /**
+ * get the primary partition id
+ * @return primaryPartitionID
+ */
+ @Override
+ public String getPrimaryPartitionId() {
+ return getHeadersMap().get(SlbHeaders.PRIMARY_PARTITION_ID);
+ }
+
+ /**
+ * get map of the current headers
+ * @return Map<String, String> headers
+ */
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return getHeaders().getHeaders();
+ }
+
+ /**
+ * supplement the DPSHeaders with any specific core service headers
+ * @param input Map<String, String> of core headers
+ * @return DpsHeaders dpsHeaders
+ */
+ @Override
+ public DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
+ Preconditions.checkNotNull(input, "input headers cannot be null");
+
+ DpsHeaders output = DpsHeaders.createFromMap(input);
+
+ return output;
+ }
+
+ /**
+ * create string representing a comma delimited list of the current headers
+ * @return
+ */
+ @Override
+ public String toString() {
+ return this.getHeadersMap().entrySet().stream().filter(map -> !FORBIDDEN_FROM_LOGGING.contains(map.getKey().toLowerCase())).map(Map.Entry::toString).collect(Collectors.joining(" | "));
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
new file mode 100644
index 0000000000000000000000000000000000000000..c7f0eb4edf857339b2872bee6c52b832d0ee92aa
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -0,0 +1,61 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.util;
+
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.SendMessageRequest;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+
+@Primary
+@Component
+public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
+
+ private AmazonSQS sqsClient;
+
+ @Value("${aws.region}")
+ private String region;
+
+ @Value("${aws.sqs.queue}")
+ private String queueName;
+
+ @Inject
+ public void init() {
+ AmazonSQSConfig sqsConfig = new AmazonSQSConfig(region);
+ sqsClient = sqsConfig.AmazonSQS();
+ }
+
+ @Override
+ public void createWorkerTask(String payload, DpsHeaders headers) {
+ createTask(payload, headers);
+ }
+
+ @Override
+ public void createReIndexTask(String payload,DpsHeaders headers) {
+ createTask(payload, headers);
+ }
+
+ private void createTask(String payload, DpsHeaders headers) {
+ String queueUrl = sqsClient.getQueueUrl(queueName).getQueueUrl();
+ SendMessageRequest messageRequest = new SendMessageRequest().withQueueUrl(queueUrl).withMessageBody(payload);
+ sqsClient.sendMessage(messageRequest);
+ }
+}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/RequestInfoImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/RequestInfoImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..9baf2c4a4845cf8ecfc1f6a042b83eed40914028
--- /dev/null
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/RequestInfoImpl.java
@@ -0,0 +1,109 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.util;
+
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.service.coreis.Preconditions;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.HashSet;
+import java.util.Map;
+
+
+@Primary
+@Log
+@Component
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders headersMap;
+
+
+ private static final HashSet<String> FORBIDDEN_FROM_LOGGING = new HashSet<>();
+ static {
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.AUTHORIZATION);
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.ON_BEHALF_OF);
+ }
+
+ /**
+ * Get list of current headers
+ * @return DpsHeaders headers
+ */
+ @Override
+ public DpsHeaders getHeaders() {
+ if (headersMap == null) {
+ log.warning("Headers Map DpsHeaders is null");
+ // throw to prevent null reference exception below
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid Headers", "Headers Map DpsHeaders is null");
+ }
+ DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders());
+ return headers;
+ }
+
+ /**
+ * get partition id and fallback to account id
+ * @return Partition ID
+ */
+ @Override
+ public String getPartitionId() {
+ return getHeaders().getPartitionIdWithFallbackToAccountId();
+ }
+
+ /**
+ * get map of the current headers
+ * @return Map<String, String> headers
+ */
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return getHeaders().getHeaders();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersMap();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ return getHeaders();
+ }
+
+
+ private DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
+ Preconditions.checkNotNull(input, "input headers cannot be null");
+
+ DpsHeaders output = DpsHeaders.createFromMap(input);
+
+ return output;
+ }
+
+ @Override
+ public boolean isCronRequest() {
+ return false;
+ }
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ return false;
+ }
+}
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..7d9bfebe0d61ac3aa6cb31d7c1d751de156b5022
--- /dev/null
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -0,0 +1,73 @@
+server.servlet.contextPath=/api/indexer/v2/
+logging.level.org.springframework.web=DEBUG
+server.port=${APPLICATION_PORT}
+JAVA_HEAP_OPTS=-Xms${JAVA_HEAP_MEMORY}M -Xmx${JAVA_HEAP_MEMORY}M
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+aws.threads=50
+DEFAULT_DATA_COUNTRY=US
+CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
+CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
+
+## AWS DynamoDB configuration
+aws.dynamodb.key=kind
+aws.dynamodb.table.prefix=${ENVIRONMENT}-
+aws.dynamodb.region=${AWS_REGION}
+aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
+
+## AWS S3 configuration
+aws.s3.region=${AWS_REGION}
+aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
+aws.s3.records.bucket-name=${ENVIRONMENT}-${S3_DATA_BUCKET}
+aws.s3.max-record-threads=2000
+aws.s3.enable-https=true
+
+## AWS SNS configuration
+aws.sns.region=${AWS_REGION}
+aws.sns.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${ENVIRONMENT}-${SNS_TOPIC_NAME}
+aws.sns.topic-name=${ENVIRONMENT}-${SNS_TOPIC_NAME}
+
+## AWS SQS Configuration
+aws.sqs.queue=${ENVIRONMENT}-osdu-indexer-queue
+
+#Spring Configuration
+spring.security.user.name=opendes@byoc.local
+spring.security.user.password=123
+spring.security.user.roles=service.storage.admin
+
+
+# AWS ES configuration
+aws.es.host=https://search-${ENVIRONMENT}-osdu-indexer-i5bpf2gv4iv6ha2xi7rook2rga.${AWS_REGION}.es.amazonaws.com
+aws.es.port=-1
+aws.es.userNameAndPassword=notused
+aws.region=${AWS_REGION}
+aws.es.serviceName=es
+
+#aws.es.host=fc7169d00c9e437993904a989102d662.${AWS_REGION}.aws.found.io
+#aws.es.port=9243
+#aws.es.userNameAndPassword=elastic:YcfDGHnc2SpKmRfwTb6PBQcK
+
+GAE_SERVICE=indexer
+
+STORAGE_SCHEMA_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+INDEXER_QUEUE_HOST=http://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${ENVIRONMENT}-osdu-indexer-queue
+
+## AWS ElastiCache configuration
+aws.elasticache.cluster.index.endpoint=${CACHE_CLUSTER_INDEX_ENDPOINT}
+aws.elasticache.cluster.index.port=${CACHE_CLUSTER_INDEX_PORT}
+aws.elasticache.cluster.schema.endpoint=${CACHE_CLUSTER_SCHEMA_ENDPOINT}
+aws.elasticache.cluster.schema.port=${CACHE_CLUSTER_SCHEMA_PORT}
+
+## Cache Settings
+aws.elasticache.cluster.index.expiration=60
+aws.elasticache.cluster.schema.expiration=60
+
+
+# Maximum size of cache value
+MAX_CACHE_VALUE_SIZE=1000
+
+## AWS Lambda configuration
+aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction-1DTM5841SUIFO
\ No newline at end of file
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/publish/PublisherImplTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/publish/PublisherImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..08fb01e57ade05e967943db33bf92a32c1648908
--- /dev/null
+++ b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/publish/PublisherImplTest.java
@@ -0,0 +1,79 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.publish;
+
+import com.amazonaws.services.sns.AmazonSNS;
+import com.amazonaws.services.sns.model.MessageAttributeValue;
+import com.amazonaws.services.sns.model.PublishRequest;
+import com.amazonaws.services.sns.model.PublishResult;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
+import org.opengroup.osdu.indexer.aws.IndexerAwsApplication;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+
+@RunWith(MockitoJUnitRunner.class)
+@SpringBootTest(classes = {IndexerAwsApplication.class})
+public class PublisherImplTest {
+
+ @InjectMocks
+ private PublisherImpl publisher = new PublisherImpl();
+
+ @Mock
+ AmazonSNS snsClient;
+
+ public void setUp() {
+ snsClient = mock(AmazonSNS.class);
+ }
+
+ @Test
+ public void publishStatusChangedTagsToTopic() throws Exception {
+ // Arrange
+ DpsHeaders headers = new DpsHeaders();
+ JobStatus jobStatus = new JobStatus();
+ Mockito.when(snsClient.publish(Mockito.any(PublishRequest.class)))
+ .thenReturn(Mockito.any(PublishResult.class));
+
+ Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ headers.addCorrelationIdIfMissing();
+ messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getCorrelationId()));
+
+ PublishRequest publishRequest = new PublishRequestBuilder().generatePublishRequest("data", jobStatus.getStatusesList(), messageAttributes,null);
+ // Act
+ publisher.publishStatusChangedTagsToTopic(headers, jobStatus);
+
+ // Assert
+ Mockito.verify(snsClient, Mockito.times(1)).publish(Mockito.eq(publishRequest));
+ }
+}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..805e9c8cf2b72c2b003404203b66a26ecb4f58c9
--- /dev/null
+++ b/testing/indexer-test-aws/pom.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright © Amazon Web Services
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-aws</artifactId>
+ <version>0.0.1</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <cucumber.version>1.2.5</cucumber.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- Internal packages -->
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-core</artifactId>
+ <version>0.0.1</version>
+ </dependency>
+
+ <!-- AWS specific packages -->
+ <dependency>
+ <groupId>org.opengroup.osdu.core.aws</groupId>
+ <artifactId>aws-osdu-util</artifactId>
+ <version>0.0.4</version>
+ </dependency>
+
+ <!-- AWS managed packages -->
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-request-signing-apache-interceptor</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Testing -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-java</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-junit</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- third party Apache 2.0 license packages -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.json</artifactId>
+ <version>1.1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.19.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.1-jre</version>
+ </dependency>
+
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!--Logging-->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.11.2</version>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.5</version>
+ <configuration>
+ <systemPropertyVariables>
+ <AWS_COGNITO_CLIENT_ID>3rmgmg8mup281ttc1mbut1pimc</AWS_COGNITO_CLIENT_ID>
+ <AWS_COGNITO_AUTH_FLOW>USER_PASSWORD_AUTH</AWS_COGNITO_AUTH_FLOW>
+ <AWS_COGNITO_AUTH_PARAMS_USER>test-user-with-access@testing.com</AWS_COGNITO_AUTH_PARAMS_USER>
+ <AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS>test-user-without-access@testing.com</AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS>
+ <AWS_COGNITO_AUTH_PARAMS_PASSWORD>Password123*</AWS_COGNITO_AUTH_PARAMS_PASSWORD>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordAncestry.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
similarity index 52%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordAncestry.java
rename to testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
index feb488b33bae23942a63db03a68aae5a41b78f75..e6887ea5ef12afed8d4d7904796ae70d6a4ccada 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordAncestry.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,12 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.step_definitions.index.record;
-import lombok.Data;
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
-@Data
-public class RecordAncestry {
-
- private String[] parents;
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexrecord/IndexRecord.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
+public class RunTest {
}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..83c36a2456d3ecbe1284db5c1ded925d49d731aa
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -0,0 +1,67 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.step_definitions.index.record;
+
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.util.AWSHTTPClient;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.DataTable;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.util.ElasticUtilsAws;
+
+@Log
+public class Steps extends RecordSteps {
+
+ public Steps() {
+ super(new AWSHTTPClient(), new ElasticUtilsAws());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new AWSHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
+ }
+
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
+ }
+
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordStatus.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/AWSHTTPClient.java
similarity index 51%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordStatus.java
rename to testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/AWSHTTPClient.java
index 80aad1bc1437a0131d23814489353680a8d00937..a6951f765bd4a24b43adf0d591d700e1dc4fdbbd 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordStatus.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/AWSHTTPClient.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,28 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.util;
-import lombok.Builder;
-import lombok.Data;
import lombok.ToString;
+import lombok.extern.java.Log;
-@Data
-@Builder
-public class RecordStatus {
+import java.io.IOException;
- private String id;
- private String kind;
- private String operationType;
+@Log
+@ToString
+public class AWSHTTPClient extends HTTPClient {
- private IndexingStatus status;
+ private static String token = null;
- @ToString.Exclude private IndexProgress indexProgress;
-
- public String getLatestTrace() {
- if (indexProgress != null && indexProgress.getTrace() != null && indexProgress.getTrace().size() > 0) {
- return indexProgress.getTrace().peek();
+ @Override
+ public synchronized String getAccessToken() {
+ if(token == null) {
+ try {
+ token = "Bearer " + JwtTokenUtil.getAccessToken();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
- return null;
+ return token;
}
}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
new file mode 100644
index 0000000000000000000000000000000000000000..fff624c682d59709622511e86e0ce499191f8ff1
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -0,0 +1,37 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import com.amazonaws.auth.AWS4Signer;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequestInterceptor;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.aws.iam.IAMConfig;
+
+public class ElasticUtilsAws extends ElasticUtils {
+
+ @Override
+ protected RestHighLevelClient createClient(String username, String password, String host) {
+ AWSCredentialsProvider credentials = new IAMConfig().amazonAWSCredentials();
+ AWS4Signer signer = new AWS4Signer();
+ signer.setServiceName(username);
+ signer.setRegionName(password);
+ HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(username, signer, credentials);
+ return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor)));
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Schema.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
similarity index 50%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Schema.java
rename to testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
index 6bbe44372cf15031fb04d079b216659085695589..3ff2b4e3166783fb6d44b1fd5ccba19055a237b4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Schema.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -1,4 +1,4 @@
-// Copyright 2017-2019, Schlumberger
+// Copyright © Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,28 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.util;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
+import org.opengroup.osdu.core.aws.cognito.AWSCognitoClient;
-import java.util.List;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Schema {
-
- private String kind;
- private List<Mapping> schema;
-
- @Data
- @Builder
- public static class Mapping {
- private String path;
- private String kind;
+class JwtTokenUtil {
+ static String getAccessToken() {
+ String clientId = Config.getAWSCognitoClientId();
+ String authFlow = Config.getAWSCognitoAuthFlow();
+ String user = Config.getAWSCognitoUser();
+ String password = Config.getAWSCognitoPassword();
+ AWSCognitoClient client = new AWSCognitoClient(clientId, authFlow, user, password);
+ return client.getTokenForUserWithAccess();
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/DeleteSchemaSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/DeleteSchemaSteps.java
deleted file mode 100644
index a6a9404c68bf35b90955d43ea05f376a98ebf79f..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/DeleteSchemaSteps.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package org.opengroup.osdu.common;
-
-import com.sun.jersey.api.client.ClientResponse;
-
-import cucumber.api.DataTable;
-
-import org.opengroup.osdu.models.Setup;
-import org.opengroup.osdu.models.TestIndex;
-import org.opengroup.osdu.response.ErrorResponseMock;
-import org.opengroup.osdu.util.Config;
-import org.opengroup.osdu.util.HTTPClient;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-
-public class DeleteSchemaSteps extends TestsBase {
-
- private static String timeStamp = String.valueOf(System.currentTimeMillis());
-
- private Map<String, String> headers = httpClient.getCommonHeader();
-
- private static boolean dunit = false;
-
- private String deleteUrl;
-
- private Map<String, TestIndex> inputRecordMap = new HashMap<>();
-
- public DeleteSchemaSteps(HTTPClient httpClient) {
- super(httpClient);
- }
-
- /******************One time cleanup for whole feature**************/
- public void tearDown() {
- for (String kind : inputRecordMap.keySet()) {
- TestIndex testIndex = inputRecordMap.get(kind);
- testIndex.cleanupIndex();
- }
- }
-
- public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
- List<Setup> inputlist = dataTable.asList(Setup.class);
- for (Setup input : inputlist) {
- TestIndex testIndex = new TestIndex();
- testIndex.setHttpClient(httpClient);
- testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
- testIndex.setKind(generateActualName(input.getKind(), timeStamp));
- testIndex.setMappingFile(input.getMappingFile());
- testIndex.setRecordFile(input.getRecordFile());
- inputRecordMap.put(testIndex.getKind(), testIndex);
- }
- /******************One time setup for whole feature**************/
- if (!dunit) {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- tearDown();
- }
- });
- dunit = true;
- for (String kind : inputRecordMap.keySet()) {
- TestIndex testIndex = inputRecordMap.get(kind);
- testIndex.setupIndex();
- }
- }
- }
-
- public void i_send_a_delete_request_with(String kind) throws Throwable {
- String actualKind = generateActualName(kind, timeStamp);
- deleteUrl = String.format(this.getApi(), actualKind);
- }
-
- public void the_index_should_get_delete_and_I_should_get_response(int code) throws Throwable {
- ClientResponse clientResponse = executeGetRequest(deleteUrl, headers, httpClient.getAccessToken());
- assertEquals(code, clientResponse.getStatus());
- }
-
- public void i_should_get_response_with_reason_message_and_errors(List<Integer> codes, String type, String msg,
- String error) throws Throwable {
- ErrorResponseMock response = executeQuery(deleteUrl, null, headers, httpClient.getAccessToken(), ErrorResponseMock.class);
- assertTrue(codes.contains(response.getResponseCode()));
- if (response.getErrors() != null) {
- error = generateActualName(error, timeStamp);
- assertEquals(generateActualName(error,timeStamp), response.getErrors().get(0));
- }
- assertNotNull(response.getMessage());
- assertNotNull(response.getReason());
- assertEquals(type.toLowerCase(), response.getReason().toLowerCase());
- assertEquals(generateActualName(msg,timeStamp), response.getMessage());
- }
-
- @Override
- protected String getHttpMethod() {
- return "DELETE";
- }
-
- @Override
- protected String getApi() {
- return Config.getSearchBaseURL() + "index/%s";
- }
-}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/GetSchemaSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/GetSchemaSteps.java
deleted file mode 100644
index 5031153ebd8200657ce8929521cac995979ff1cb..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/GetSchemaSteps.java
+++ /dev/null
@@ -1,109 +0,0 @@
-package org.opengroup.osdu.common;
-
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-
-import org.opengroup.osdu.models.Setup;
-import org.opengroup.osdu.models.TestIndex;
-import org.opengroup.osdu.response.ErrorResponseMock;
-import org.opengroup.osdu.util.Config;
-import org.opengroup.osdu.util.HTTPClient;
-
-import com.sun.jersey.api.client.ClientResponse;
-import cucumber.api.DataTable;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-public class GetSchemaSteps extends TestsBase {
-
- private static String timeStamp = String.valueOf(System.currentTimeMillis());
- private static boolean dunit = false;
- private Map<String, String> headers = httpClient.getCommonHeader();
- private Map<String, TestIndex> inputRecordMap = new HashMap<>();
- private String schemaUrl;
-
- public GetSchemaSteps(HTTPClient httpClient) {
- super(httpClient);
- }
-
-
- /******************One time cleanup for whole feature**************/
- public void tearDown() {
- for (String kind : inputRecordMap.keySet()) {
- TestIndex testIndex = inputRecordMap.get(kind);
- testIndex.cleanupIndex();
- }
- }
-
- public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
- List<Setup> inputlist = dataTable.asList(Setup.class);
- for (Setup input : inputlist) {
- TestIndex testIndex = new TestIndex();
- testIndex.setHttpClient(httpClient);
- testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
- testIndex.setKind(generateActualName(input.getKind(), timeStamp));
- testIndex.setMappingFile(input.getMappingFile());
- inputRecordMap.put(testIndex.getKind(), testIndex);
- }
- /******************One time setup for whole feature**************/
- if (!dunit) {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- tearDown();
- }
- });
- dunit = true;
- for (String kind : inputRecordMap.keySet()) {
- TestIndex testIndex = inputRecordMap.get(kind);
- testIndex.addIndex();
- }
- }
- }
-
- public void i_send_get_schema_request_with(String kind) throws Throwable {
- String actualKind = generateActualName(kind, timeStamp);
- schemaUrl = String.format(this.getApi(), actualKind);
- }
-
- public void i_send_request_to_tenant(String tenant) throws Throwable {
- headers = HTTPClient.overrideHeader(headers, getTenantMapping(tenant));
- }
-
- public void i_should_get_response_with_reason_message_and_errors(int responseCode, String type, String msg,
- String error) throws Throwable {
-
- ErrorResponseMock response = executeQuery(schemaUrl, null, headers, httpClient.getAccessToken(), ErrorResponseMock.class);
- assertEquals(responseCode, response.getResponseCode());
- if (response.getErrors() != null) {
- error = generateActualName(error, timeStamp);
- assertEquals(generateActualName(error, timeStamp), response.getErrors().get(0));
- }
- assertEquals(type, response.getReason());
- assertEquals(generateActualName(msg, timeStamp),response.getMessage());
- }
-
- public void i_should_get_status_with_response(int statusCode, String response) throws Throwable {
-
- ClientResponse schemaResponse = executeGetRequest(schemaUrl, headers, httpClient.getAccessToken());
- assertEquals(statusCode, schemaResponse.getStatus());
- String expectedResponse = generateActualName(response, timeStamp);
- JsonObject expectedJson = new JsonParser().parse(expectedResponse).getAsJsonObject();
- JsonObject actualJson = new JsonParser().parse(schemaResponse.getEntity(String.class)).getAsJsonObject();
- assertEquals(expectedJson, actualJson);
- }
-
- @Override
- protected String getApi() {
- return Config.getSearchBaseURL() + "index/schema/%s";
- }
-
- @Override
- protected String getHttpMethod() {
- return "GET";
- }
-
-}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/MappingSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/MappingSteps.java
deleted file mode 100644
index 47e6fdbb215673724f557695f957a3981de170ec..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/MappingSteps.java
+++ /dev/null
@@ -1,114 +0,0 @@
-package org.opengroup.osdu.common;
-
-import com.google.api.client.http.HttpMethods;
-import com.google.gson.Gson;
-import com.sun.jersey.api.client.ClientResponse;
-import cucumber.api.DataTable;
-import lombok.extern.java.Log;
-
-import java.util.*;
-
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.opengroup.osdu.models.Setup;
-import org.opengroup.osdu.request.Query;
-import org.opengroup.osdu.response.ResponseMock;
-import org.opengroup.osdu.util.Config;
-import org.opengroup.osdu.util.ElasticUtils;
-import org.opengroup.osdu.util.HTTPClient;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-@Log
-public class MappingSteps extends TestsBase {
- private Map<String, String> headers;
- private String fieldName;
- private String timeStamp;
- private static boolean dunit = false;
- private Query requestQuery = new Query();
- private ElasticUtils elasticUtils = new ElasticUtils();
- private Map<String, Object> requestPayload = new HashMap<>();
-
- private static String updateIndexMappingUrl = Config.getIndexerBaseURL() + "kinds";
- private static String searchQeueryURL=Config.getSearchBaseURL() + "query";
-
-
- public MappingSteps(HTTPClient httpClient) {
- super(httpClient);
- headers = httpClient.getCommonHeader();
- fieldName="";
- timeStamp = String.valueOf(System.currentTimeMillis());
- }
-
- public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) {
- if (!dunit) {
- List<Setup> inputlist = dataTable.asList(Setup.class);
- setUp(inputlist, timeStamp);
- dunit = true;
- }
- }
-
- public void i_update_in_to_enable_multifield_indexing(String fieldNameVal, String index1, String index2) throws Throwable {
- Set<String> indices = new HashSet<>();
- System.out.println("Indices "+ index1 + index2);
- indices.add(generateActualName(index1,timeStamp));
- indices.add(generateActualName(index2,timeStamp));
- fieldName=fieldNameVal;
- requestPayload.put("indices", indices);
- requestPayload.put("operator", "keyword");
- }
-
- public void i_send_request_to_tenant(String tenant) {
- headers = HTTPClient.overrideHeader(headers, getTenantMapping(tenant));
- }
-
- public void i_should_get_response(int responseCode) throws Throwable {
- String payload = new Gson().toJson(requestPayload);
- ClientResponse clientResponse = httpClient.send(HttpMethods.PUT, this.getApi()+"/"+this.fieldName, payload, headers, httpClient.getAccessToken());
- assertEquals(responseCode, clientResponse.getStatus());
- }
-
- public void i_send_with(String query, String kind) {
- requestQuery.setQuery(query);
- requestQuery.setKind(generateActualName(kind, timeStamp));
- }
-
- public void i_send_None_with(String kind) {
- requestQuery.setKind(generateActualName(kind, timeStamp));
- }
-
- public void i_aggregate_by(String aggField) throws Throwable {
- requestQuery.setAggregateBy(aggField+".keyword");
- }
-
- public void i_should_get_in_response_records(int resultCount) {
- String payload = requestQuery.toString();
- ResponseMock response = executeQuery(searchQeueryURL, payload, this.headers, httpClient.getAccessToken(), ResponseMock.class);
- assertEquals(200, response.getResponseCode());
- assertEquals(resultCount, response.getResults().size());
- }
-
- public void i_want_to_validate_mapping(String indexOne, String indexTwo,String fieldName,String type) throws Throwable {
- ImmutableOpenMap<String, MappingMetaData> elasticMapping = elasticUtils.getMapping(generateActualName(indexOne,timeStamp));
- assertNotNull(elasticMapping);
- MappingMetaData typeMapping = elasticMapping.get(type);
- Map<String, Object> mapping = typeMapping.sourceAsMap();
- assertNotNull(mapping);
- String mappingJson = new Gson().toJson(mapping);
- System.out.println(mappingJson);
- assertTrue(mappingJson.contains(fieldName));
- assertTrue(mappingJson.contains("raw"));
- }
-
- @Override
- protected String getApi() {
- return updateIndexMappingUrl;
- }
-
- @Override
- protected String getHttpMethod() {
- return "POST";
- }
-}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 9dbfc60604e9f28e711b1c55c4949633bc91d603..fe57b8cc49e8566c72ed28e4194fe666a4f886d7 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -33,14 +33,16 @@ public class RecordSteps extends TestsBase {
private Map<String, TestIndex> inputIndexMap = new HashMap<>();
private boolean shutDownHookAdded = false;
- private String timeStamp;
+ private String timeStamp = String.valueOf(System.currentTimeMillis());
private List<Map<String, Object>> records;
- private Map<String, String> headers;
+ private Map<String, String> headers = httpClient.getCommonHeader();
public RecordSteps(HTTPClient httpClient) {
super(httpClient);
- timeStamp = String.valueOf(System.currentTimeMillis());
- headers = httpClient.getCommonHeader();
+ }
+
+ public RecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
}
/******************One time cleanup for whole feature**************/
@@ -63,7 +65,7 @@ public class RecordSteps extends TestsBase {
List<Setup> inputList = dataTable.asList(Setup.class);
for (Setup input : inputList) {
- TestIndex testIndex = new TestIndex();
+ TestIndex testIndex = getTextIndex();
testIndex.setHttpClient(httpClient);
testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
testIndex.setKind(generateActualName(input.getKind(), timeStamp));
@@ -117,7 +119,6 @@ public class RecordSteps extends TestsBase {
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
index = generateActualName(index, timeStamp);
- ElasticUtils elasticUtils = new ElasticUtils();
ImmutableOpenMap<String, MappingMetaData> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
@@ -129,14 +130,12 @@ public class RecordSteps extends TestsBase {
public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
index = generateActualName(index, timeStamp);
- ElasticUtils elasticUtils = new ElasticUtils();
long numOfIndexedDocuments = createIndex(index);
long documentCountByQuery = elasticUtils.fetchRecordsByExistQuery(index, skippedAttributes);
assertEquals(expectedCount, documentCountByQuery);
}
private long createIndex(String index) throws InterruptedException, IOException {
- ElasticUtils elasticUtils = new ElasticUtils();
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
index 89fbb2a05f445d4075f93c81883620bc98c1acf5..43cd1b4d429d69acd6486d418fb93795d03b3319 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
@@ -6,6 +6,7 @@ import org.opengroup.osdu.models.Legal;
import org.opengroup.osdu.models.Setup;
import org.opengroup.osdu.models.TestIndex;
import org.opengroup.osdu.response.ResponseBase;
+import org.opengroup.osdu.util.ElasticUtils;
import org.opengroup.osdu.util.HTTPClient;
import com.sun.jersey.api.client.ClientResponse;
@@ -28,17 +29,33 @@ public abstract class TestsBase {
protected Scenario scenario;
protected Map<String, String> tenantMap = new HashMap<>();
protected Map<String, TestIndex> inputRecordMap = new HashMap<>();
+ protected ElasticUtils elasticUtils;
public TestsBase(HTTPClient httpClient) {
this.httpClient = httpClient;
tenantMap.put("tenant1", getDataPartitionIdTenant1());
tenantMap.put("tenant2", getDataPartitionIdTenant2());
tenantMap.put("common", "common");
+
+ elasticUtils = new ElasticUtils();
+ }
+
+ public TestsBase(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ this.httpClient = httpClient;
+ tenantMap.put("tenant1", getDataPartitionIdTenant1());
+ tenantMap.put("tenant2", getDataPartitionIdTenant2());
+ tenantMap.put("common", "common");
+
+ this.elasticUtils = elasticUtils;
+ }
+
+ protected TestIndex getTextIndex(){
+ return new TestIndex(elasticUtils);
}
protected void setUp(List<Setup> inputList, String timeStamp) {
for (Setup input : inputList) {
- TestIndex testIndex = new TestIndex();
+ TestIndex testIndex = getTextIndex();
testIndex.setHttpClient(httpClient);
testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
testIndex.setKind(generateActualName(input.getKind(), timeStamp));
@@ -87,26 +104,12 @@ public abstract class TestsBase {
protected abstract String getHttpMethod();
- protected String executeQuery(String api, String payLoad, Map<String, String> headers, String token) {
- ClientResponse clientResponse = httpClient.send(this.getHttpMethod(), api, payLoad, headers, token);
- logCorrelationIdWithFunctionName(clientResponse.getHeaders());
- log.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
- assertEquals(MediaType.APPLICATION_JSON, clientResponse.getType().toString());
- return clientResponse.getEntity(String.class);
- }
-
protected <T extends ResponseBase> T executeQuery(String api, String payLoad, Map<String, String> headers, String token, Class<T> typeParameterClass) {
ClientResponse clientResponse = httpClient.send(this.getHttpMethod(), api, payLoad, headers, token);
logCorrelationIdWithFunctionName(clientResponse.getHeaders());
return getResponse(clientResponse, typeParameterClass);
}
- protected <T extends ResponseBase> T executeQuery(String payLoad, Map<String, String> headers, String token, Class<T> typeParameterClass) {
- ClientResponse clientResponse = httpClient.send(this.getHttpMethod(), this.getApi(), payLoad, headers, token);
- logCorrelationIdWithFunctionName(clientResponse.getHeaders());
- return getResponse(clientResponse, typeParameterClass);
- }
-
private <T extends ResponseBase> T getResponse(ClientResponse clientResponse, Class<T> typeParameterClass) {
log.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
assertEquals(MediaType.APPLICATION_JSON, clientResponse.getType().toString());
@@ -118,18 +121,6 @@ public abstract class TestsBase {
return response;
}
- protected ClientResponse executeGetRequest(String api, Map<String, String> headers, String token) {
- return executeRequest(this.getHttpMethod(), api, headers, token);
- }
-
- protected ClientResponse executeRequest(String method, String api, Map<String, String> headers, String token) {
- ClientResponse clientResponse = httpClient.send(method, api, null, headers, token);
- if (clientResponse.getType() != null) {
- log.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
- }
- logCorrelationIdWithFunctionName(clientResponse.getHeaders());
- return clientResponse;
- }
private void logCorrelationIdWithFunctionName(MultivaluedMap<String, String> headers) {
log.info(String.format("Scenario Name: %s, Correlation-Id: %s", scenario.getId(), headers.get("correlation-id")));
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java
index 00ac7b1c9bd2fe9780c5d2f40baf761e8aa429c5..d8c611d8b5a3aff19a5ab23606f45476d9f57609 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import static org.opengroup.osdu.util.Config.*;
@Data
-@NoArgsConstructor
public class TestIndex {
private static final Logger LOGGER = Logger.getLogger(TestIndex.class.getName());
private String kind;
@@ -35,9 +34,13 @@ public class TestIndex {
private String[] ownerGroup;
private HTTPClient httpClient;
private Map<String, String> headers;
- private ElasticUtils elasticUtils = new ElasticUtils();
+ private ElasticUtils elasticUtils;
private Gson gson = new Gson();
+ public TestIndex(ElasticUtils elasticUtils){
+ this.elasticUtils = elasticUtils;
+ }
+
public void setHttpClient(HTTPClient httpClient) {
this.httpClient = httpClient;
headers = httpClient.getCommonHeader();
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
index 1c005e6fbf81f44441ba2edbba91d40550917067..e22b4fac2cdc0484867357610985cee814a6502f 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
@@ -83,6 +83,22 @@ public class Config {
return getEnvironmentVariableOrDefaultValue("ENTITLEMENTS_DOMAIN", DEFAULT_ENTITLEMENTS_DOMAIN);
}
+ public static String getAWSCognitoClientId() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_CLIENT_ID", "");
+ }
+
+ public static String getAWSCognitoAuthFlow() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_FLOW", "");
+ }
+
+ public static String getAWSCognitoUser() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_USER", "");
+ }
+
+ public static String getAWSCognitoPassword() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_PASSWORD", "");
+ }
+
private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
String environmentVariable = getEnvironmentVariable(key);
if (environmentVariable == null) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 8cf151f60594a4c09d16e45f287dcc78d6f73438..d817d347b84116df0551460e8cddf205820e8ea2 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -78,7 +78,7 @@ public class ElasticUtils {
public void createIndex(String index, String mapping) {
try {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1).build();
@@ -131,7 +131,7 @@ public class ElasticUtils {
BulkResponse bulkResponse = null;
try {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("Done creating records inside index with name: " + index);
}
@@ -145,7 +145,7 @@ public class ElasticUtils {
}
try {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
RefreshRequest request = new RefreshRequest(index);
RefreshResponse refreshResponse = client.indices().refresh(request, RequestOptions.DEFAULT);
log.info(String.format("refreshed index, acknowledged shards: %s | failed shards: %s | total shards: %s ", refreshResponse.getSuccessfulShards(), refreshResponse.getFailedShards(), refreshResponse.getTotalShards()));
@@ -158,7 +158,7 @@ public class ElasticUtils {
}
public void deleteIndex(String index) {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
//retry if the elastic cluster is snapshotting and we cant delete it
for (int retries = 0; ; retries++) {
try {
@@ -194,7 +194,7 @@ public class ElasticUtils {
public long fetchRecords(String index) throws IOException {
try {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest request = new SearchRequest(index);
SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
return searchResponse.getHits().totalHits;
@@ -207,7 +207,7 @@ public class ElasticUtils {
public long fetchRecordsByExistQuery(String index, String attributeName) throws IOException {
try {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.existsQuery(attributeName));
@@ -223,7 +223,7 @@ public class ElasticUtils {
}
public ImmutableOpenMap<String, MappingMetaData> getMapping(String index) throws IOException {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
request.indices(index);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
@@ -233,7 +233,7 @@ public class ElasticUtils {
}
public void refreshIndex(String index) throws IOException {
- try (RestHighLevelClient client = ElasticUtils.createClient(username, password, host)) {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
try {
RefreshRequest request = new RefreshRequest(index);
client.indices().refresh(request, RequestOptions.DEFAULT);
@@ -272,7 +272,8 @@ public class ElasticUtils {
return dataList;
}
- private static RestHighLevelClient createClient(String username, String password, String host) {
+
+ protected RestHighLevelClient createClient(String username, String password, String host) {
RestHighLevelClient restHighLevelClient;
int port = Config.PORT;
diff --git a/testing/indexer-test-core/src/main/resources/features/delete/Delete.feature b/testing/indexer-test-core/src/main/resources/features/delete/Delete.feature
deleted file mode 100644
index 519b648fa6840c9e320ffa1739a9e4723fe47030..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/delete/Delete.feature
+++ /dev/null
@@ -1,27 +0,0 @@
-Feature: Delete search indexes
- If a user wants to delete any index, search should offer ways to do the same.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile |
- | tenant1:testdelete<timestamp>:well:1.0.0 | tenant1-testdelete<timestamp>-well-1.0.0 | records_1 | records_1 |
- | tenant1:testdelete<timestamp>:well:2.0.0 | tenant1-testdelete<timestamp>-well-2.0.0 | records_2 | records_2 |
-
-
- Scenario Outline: Delete a given index from the Search
- When I send a delete request with <kind>
- Then the index should get delete and I should get <response_code> response
-
- Examples:
- | kind | response_code |
- | "tenant1:testdelete<timestamp>:well:1.0.0" | 200 |
-
- Scenario Outline: Fail the request for deletion of a index from the Search with invalid inputs
- When I send a delete request with <kind>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | kind | response_code | reponse_type | response_message | errors |
- | "tenant1:testdelete<timestamp>:*:*" | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Not a valid record kind. Found: tenant1:testdelete<timestamp>:*:*, required format is partition:data-source-id:type:schema-version with no wildcards e.g. tenant:well:wellbore:1.0.2" |
- | "tenant1:testdatasource:wellrating:9.0.0" | 404 | "Index deletion error" | "Kind tenant1:testdatasource:wellrating:9.0.0 not found" | "" |
-
diff --git a/testing/indexer-test-core/src/main/resources/features/kindschema/KindSchema.feature b/testing/indexer-test-core/src/main/resources/features/kindschema/KindSchema.feature
deleted file mode 100644
index ee526ad538bbb85f6e02e182d6e009508fb0e6a2..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/kindschema/KindSchema.feature
+++ /dev/null
@@ -1,26 +0,0 @@
-Feature: Get schema for a given kind
- Allow user to find the attributes indexed and their respective data types.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile |
- | tenant1:testschema<timestamp>:well:1.0.0 | tenant1-testschema<timestamp>-well-1.0.0 | records_1 |
-
- Scenario Outline: Get a schema from search for a kind
- When I send get schema request with <kind>
- And I send request to tenant <tenant>
- Then I should get <response_code> status with response <response_message>
-
- Examples:
- | tenant | kind | response_code | response_message |
- | "tenant1" | "tenant1:testschema<timestamp>:well:1.0.0" | 200 | "{"tenant1-testschema<timestamp>-well-1.0.0":{"mappings":{"well":{"properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"kind":{"type":"keyword"},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"keyword"},"x-acl":{"type":"keyword"}}}}}}" |
-
- Scenario Outline: Fail request to get schema from search with invalid inputs
- When I send get schema request with <kind>
- And I send request to tenant <tenant>
- Then I should get <response_code> response with reason: <response_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | tenant | kind | response_code | response_type | response_message | errors |
- | "tenant1" | "tenant1-testschema<timestamp>:*:*" | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Not a valid record kind. Found: tenant1-testschema<timestamp>:*:*, required format is partition:data-source-id:type:schema-version with no wildcards e.g. tenant:well:wellbore:1.0.2" |
- | "tenant1" | "tenant1:testdatasource:wellrating:9.0.0" | 404 | "Kind not found" | "Kind tenant1:testdatasource:wellrating:9.0.0 not found" | "" |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/query/crosscluster/Query.feature b/testing/indexer-test-core/src/main/resources/features/query/crosscluster/Query.feature
deleted file mode 100644
index 4e37a677da089ca41f5bdb4d7efc87e2e6f43060..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/query/crosscluster/Query.feature
+++ /dev/null
@@ -1,103 +0,0 @@
-Feature: Search with different queries
- To allow a user to find his data quickly, search should offer multiple ways to search data.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testquery<timestamp>:well:1.0.0 | tenant1-testquery<timestamp>-well-1.0.0 | records_1 | records_1 | data.default.viewers@opendes | data.default.owners@opendes |
- | tenant1:testquery<timestamp>:well:2.0.0 | tenant1-testquery<timestamp>-well-2.0.0 | records_2 | records_2 | data.default.viewers@opendes | data.default.owners@opendes |
- | common:testquery<timestamp>:well:1.0.0 | common-testquery<timestamp>-well-1.0.0 | records_1 | records_1 | data.default.viewers@opendes | data.default.owners@opendes |
- | common:testquery<timestamp>:well:2.0.0 | common-testquery<timestamp>-well-2.0.0 | records_2 | records_2 | data.default.viewers@opendes | data.default.owners@opendes |
-
- @ignore
- Scenario Outline: Search data in a given kind
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenants <tenants>
- Then I should get in response <count> records
-
- Examples:
- | tenants | kind | query | limit | offset | returned_fields | count |
- | "tenant1","common" | "*:testquery<timestamp>:well:1.0.0" | "OSDU" | None | None | All | 6 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:well:1.0.0" | "OSDU" | None | None | All | 3 |
-
-
- @ignore
- Scenario Outline: Search data in a given a kind with invalid inputs
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I send request to tenants <tenants>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | tenants | kind | query | limit | offset | response_code | reponse_type | response_message | errors |
- | "tenant2","common" | "*:testquery<timestamp>:well:1.0.0" | None | None | None | 401 | "Access denied" | "The user is not authorized to perform this action" | "" |
-
- @ignore
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I send request to tenants <tenants>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- Then I should get in response <count> records
-
- Examples:
- | tenants | kind | query | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | count |
- | "tenant1","common" | "*:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 45 | -100 | 0 | 0 | 2 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 45 | -100 | 0 | 0 | 1 |
-
- @ignore
- Scenario Outline: Search data across the kinds with distance inputs
- When I send <query> with <kind>
- And I send request to tenants <tenants>
- And I apply geographical query on field <field>
- And define focus coordinates as (<latitude>, <longitude>) and search in a <distance> radius
- Then I should get in response <count> records
-
- Examples:
- | tenants | kind | query | field | latitude | longitude | distance | count |
- | "tenant1","common" | "*:testquery<timestamp>:well:1.0.0" | "Under development" | "data.Location" | 0 | 0 | 20000000 | 6 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:well:1.0.0" | "Under development" | "data.Location" | 0 | 0 | 20000000 | 3 |
-
- @ignore
- Scenario Outline: Search data across the kinds
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenants <tenants>
- Then I should get in response <count> records
-
- Examples:
- | tenants | kind | query | limit | offset | returned_fields | count |
- | "tenant1","common" | "*:testquery<timestamp>:*:*" | "OSDU OFFICE*" | 12 | None | All | 12 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:*:*" | "OSDU OFFICE*" | 12 | None | All | 6 |
-
-
- @ignore
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I send request to tenants <tenants>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- Then I should get in response <count> records
-
- Examples:
- | tenants | kind | query | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | count |
- | "tenant1","common" | "*:testquery<timestamp>:*:*" | None | "data.Location" | 45 | -100 | 0 | 0 | 6 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:*:*" | None | "data.Location" | 45 | -100 | 0 | 0 | 3 |
-
- @ignore
- Scenario Outline: Search data across the kinds with geo polygon inputs
- When I send <query> with <kind>
- And I send request to tenants <tenants>
- And define geo polygon with following points <points_list>
- And I apply geographical query on field <field>
- Then I should get in response <count> records
- Examples:
- | tenants | kind | query | field | points_list | count |
- | "tenant1","common" | "*:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (26.12362;-112.226716) , (26.595873;-68.457186) , (52.273184;-93.593904) | 4 |
- | "tenant1","common" | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (26.12362;-112.226716) , (26.595873;-68.457186) , (52.273184;-93.593904) | 2 |
diff --git a/testing/indexer-test-core/src/main/resources/features/query/singlecluster/Query.feature b/testing/indexer-test-core/src/main/resources/features/query/singlecluster/Query.feature
deleted file mode 100644
index 7b341d66585a55cfd057932d33e215c52d19f793..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/query/singlecluster/Query.feature
+++ /dev/null
@@ -1,205 +0,0 @@
-Feature: Search with different queries
- To allow a user to find his data quickly, search should offer multiple ways to search data.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testquery<timestamp>:well:1.0.0 | tenant1-testquery<timestamp>-well-1.0.0 | records_1 | records_1 | data.default.viewers@opendes | data.default.owners@opendes |
- | tenant1:testquery<timestamp>:well:2.0.0 | tenant1-testquery<timestamp>-well-2.0.0 | records_2 | records_2 | data.default.viewers@opendes | data.default.testowners@opendes |
-
- Scenario Outline: Search data in a given kind
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenant <tenant>
- Then I should get in response <count> records
-
- Examples:
- | tenant | kind | query | limit | offset | returned_fields | count |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | None | 0 | None | NULL | 3 |
- ######################################Range Query test cases##########################################################################
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.Rank:{1 TO 3}" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.Rank:[10 TO 20]" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.Rank:>=2" | None | None | All | 2 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "data.Established:{* TO 2012-01-01}" | None | None | All | 2 |
- ######################################Text Query test cases###########################################################################
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | "OSDU" | None | None | All | 3 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:2.0.0" | "data.OriginalOperator:OFFICE6" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | ""OFFICE2" \| OFFICE3" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:well:2.0.0" | "data.Well\*:(Data Lake Cloud)" | None | None | All | 3 |
-
- Scenario Outline: Search data in a given a kind with invalid inputs
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I send request to tenant <tenant>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | tenant | kind | query | limit | offset | response_code | reponse_type | response_message | errors |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | None | -1 | None | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'limit' must be equal or greater than 0" |
- | "tenant1" | "invalid" | None | 1 | None | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Not a valid record kind. Found: invalid" |
- | "tenant1" | "tenant1:testquery<timestamp>:well:1.0.0" | None | 1 | -1 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'offset' must be equal or greater than 0" |
- | "tenant2" | "tenant1:testquery<timestamp>:well:1.0.0" | None | None | None | 401 | "Access denied" | "The user is not authorized to perform this action" | "" |
-
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- Then I should get in response <count> records
-
- Examples:
- | kind | query | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | count |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | 45 | -100 | 0 | 0 | 2 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | 45 | -80 | 0 | 0 | 0 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 45 | -100 | 0 | 0 | 1 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 10 | -100 | 0 | 0 | 0 |
-
- Scenario Outline: Search data across the kinds with invalid bounding box inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | kind | query | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | response_code | reponse_type | response_message | errors |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 0 | 0 | 0 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "top latitude cannot be the same as bottom latitude: 0.0 == 0.0" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 0 | -100 | -10 | -100 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "left longitude cannot be the same as right longitude: -100.0 == -100.0" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 10 | -100 | 10 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "top latitude cannot be the same as bottom latitude: 10.0 == 10.0" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 45 | -100 | -95 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'latitude' value is out of the range [-90, 90]" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | 0 | -100 | 10 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "top corner is below bottom corner: 0.0 vs. 10.0" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "data.OriginalOperator:OFFICE4" | "data.Location" | None | None | 0 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Invalid payload" |
- | "tenant1:testquery<timestamp>:*:*" | None | "officeAddress" | 45 | -100 | 0 | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "" |
-
- Scenario Outline: Search data across the kinds with distance inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define focus coordinates as (<latitude>, <longitude>) and search in a <distance> radius
- Then I should get in response <count> records
-
- Examples:
- | kind | query | field | latitude | longitude | distance | count |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "Under development" | "data.Location" | 0 | 0 | 20000000 | 3 |
- | "tenant1:testquery<timestamp>:*:*" | "TEXAS OR TX" | "data.Location" | 45 | -100 | 20000000 | 2 |
-
- Scenario Outline: Search data across the kinds with invalid distance inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define focus coordinates as (<latitude>, <longitude>) and search in a <distance> radius
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | kind | query | field | latitude | longitude | distance | response_code | reponse_type | response_message | errors |
- | "tenant1:testquery<timestamp>:*:*" | "OFFICE - 2" | "data.Location" | -45 | -200 | 1000 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'longitude' value is out of the range [-180, 180]" |
- | "tenant1:testquery<timestamp>:*:*" | "TEXAS OR USA" | "data.Location" | -95 | -100 | 1000 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'latitude' value is out of the range [-90, 90]" |
- | "tenant1:testquery<timestamp>:*:*" | "Harris" | "ZipCode" | -45 | -200 | 1000 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'longitude' value is out of the range [-180, 180]" |
-
- Scenario Outline: Search data across the kinds
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the offset of starting point as <offset>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenant <tenant>
- Then I should get in response <count> records
-
- Examples:
- | tenant | kind | query | limit | offset | returned_fields | count |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | None | 1 | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | None | None | 2 | All | 4 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | None | None | None | Country | 6 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "OSDU OFFICE*" | None | None | All | 6 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "SCHLUM OFFICE" | None | None | All | 6 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | ""SCHLUM OFFICE"" | None | None | All | 0 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "data.Country:USA" | None | None | All | 2 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "TEXAS AND OFFICE3" | None | None | All | 1 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "data.OriginalOperator:OFFICE5 OR OFFICE2" | None | None | All | 2 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "data.OriginalOperator:STI OR HT" | None | None | All | 0 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "_exists_:data.Basin" | None | None | All | 4 |
- | "tenant1" | "tenant1:testquery<timestamp>:*:*" | "data.Well\*:"Data Lake Cloud"" | None | None | All | 5 |
-
-
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- Then I should get in response <count> records
-
- Examples:
- | kind | query | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | count |
- | "tenant1:testquery<timestamp>:*:*" | None | "data.Location" | 45 | -100 | 0 | 0 | 3 |
- | "tenant1:testquery<timestamp>:*:*" | None | "data.Location" | 10 | -100 | 0 | 0 | 0 |
-
- Scenario Outline: Search data across the kinds with geo polygon inputs
- When I send <query> with <kind>
- And define geo polygon with following points <points_list>
- And I apply geographical query on field <field>
- Then I should get in response <count> records
- Examples:
- | kind | query | field | points_list | count |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (26.12362;-112.226716) , (26.595873;-68.457186) , (52.273184;-93.593904) | 2 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (33.201112;-113.282863) , (33.456305;-98.269744) , (52.273184;-93.593904) | 0 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "OFFICE4" | "data.Location" | (26.12362;-112.226716) , (26.595873;-68.457186) , (52.273184;-93.593904) | 1 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (14.29056;72.18936) , (22.13762;72.18936) , (22.13762;77.18936) , (14.29056;77.18936) , (14.29056;72.18936) | 1 |
-
- Scenario Outline: Search data across the kinds with invalid geo polygon inputs
- When I send <query> with <kind>
- And define geo polygon with following points <points_list>
- And I apply geographical query on field <field>
- Then I should get <response_code> response with reason: <response_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | kind | query | field | points_list | response_code | response_type | response_message | errors |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (26.595873;-68.457186) , (52.273184;-93.593904) | 400 | "Bad Request" | "Invalid parameters were given on search request" | "too few points defined for geo polygon query" |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Location" | (516.595873;-68.457186) , (52.273184;-94.593904) , (95.273184;-93.593904) | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'latitude' value is out of the range [-90, 90]" |
-
- Scenario Outline: Search data and sort the results with the given sort fields and order
- When I send <query> with <kind>
- And I want the results sorted by <sort>
- Then I should get records in right order first record id: <first_record_id>, last record id: <last_record_id>
- Examples:
- | kind | query | sort | first_record_id | last_record_id |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["id"],"order":["ASC"]} | "test:well:1.0.0:1" | "test:well:2.0.0:3" |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["id"],"order":["DESC"]} | "test:well:2.0.0:3" | "test:well:1.0.0:1" |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["namespace","data.Rank"],"order":["ASC","DESC"]} | "test:well:1.0.0:3" | "test:well:2.0.0:1" |
-
- Scenario Outline: Search data in a given kind with invalid sort field
- When I send <query> with <kind>
- And I want the results sorted by <sort>
- Then I should get <response_code> response with reason: <response_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | kind | query | sort | response_code | response_type | response_message | errors |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":[],"order":["ASC"]} | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'sort.field' can not be null or empty" |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["id"],"order":[]} | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'sort.order' can not be null or empty" |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["id","data.Rank"],"order":["DESC"]} | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'sort.field' and 'sort.order' size do not match" |
- | "tenant1:testquery<timestamp>:well:*" | None | {"field":["id"],"order":[null]} | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Not a valid order option. It can only be either 'ASC' or 'DESC'" |
-
- Scenario Outline: Search data in a given kind with different searchAs modes
- When I send <query> with <kind>
- And I want to search as owner <is_owner>
- Then I should get in response <count> records
-
- Examples:
- | kind | query | is_owner | count |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | true | 3 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | false | 3 |
- | "tenant1:testquery<timestamp>:well:2.0.0" | None | true | 0 |
- | "tenant1:testquery<timestamp>:well:2.0.0" | None | false | 3 |
- | "tenant1:testquery<timestamp>:well:*" | None | false | 6 |
- | "tenant1:testquery<timestamp>:well:*" | None | true | 3 |
- | "tenant1:testquery<timestamp>:well:*" | "OFFICE4" | true | 1 |
- | "tenant1:testquery<timestamp>:well:*" | None | None | 6 |
-
- Scenario Outline: Search data in a given kind with aggregateBy field
- When I send <query> with <kind>
- And I want to aggregate by <aggregateBy>
- Then I should get <count> unique values
-
- Examples:
- | kind | query | aggregateBy | count |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "namespace" | 1 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "type" | 1 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | "OFFICE4" | "data.Rank" | 1 |
- | "tenant1:testquery<timestamp>:well:1.0.0" | None | "data.Rank" | 3 |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/querybycursor/crosscluster/QueryByCursor.feature b/testing/indexer-test-core/src/main/resources/features/querybycursor/crosscluster/QueryByCursor.feature
deleted file mode 100644
index 5e078b7249c9c33c67675ddab0b7ba33e191803d..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/querybycursor/crosscluster/QueryByCursor.feature
+++ /dev/null
@@ -1,56 +0,0 @@
-Feature: Search recursively on cursor with different queries
- To allow a user to find his data quickly, search should offer multiple ways to search data and iterate over all the results.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testcursor<timestamp>:well:1.0.0 | tenant1-testcursor<timestamp>-well-1.0.0 | records_1 | records_1 | data.search.integrationtest@tenant1 | data.search.integrationtest@tenant1 |
- | tenant1:testcursor<timestamp>:well:2.0.0 | tenant1-testcursor<timestamp>-well-2.0.0 | records_2 | records_2 | data.search.integrationtest@tenant1 | data.search.integrationtest@tenant1 |
- | common:testcursor<timestamp>:well:1.0.0 | common-testcursor<timestamp>-well-1.0.0 | records_1 | records_1 | data.search.integrationtest@common | data.search.integrationtest@common |
- | common:testcursor<timestamp>:well:2.0.0 | common-testcursor<timestamp>-well-2.0.0 | records_2 | records_2 | data.search.integrationtest@common | data.search.integrationtest@common |
-
- @ignore
- Scenario Outline: Search recursively page by page data across the kinds
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenants <q1_tenants>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenants <q2_tenants>
- Then I should get in response <final_count> records
-
- Examples:
- | q1_tenants | q2_tenants | kind | query | limit | returned_fields | first_count | final_count |
- | "tenant1","common" | "tenant1","common" | "*:testcursor<timestamp>:*:*" | "TX OR TEXAS OR FRANCE" | 3 | All | 3 | 3 |
- | "tenant1","common" | "tenant1","common" | "tenant1:testcursor<timestamp>:*:*" | "TX OR TEXAS OR FRANCE" | 3 | All | 3 | 0 |
-
-
- @ignore
- Scenario Outline: Search recursively page by page data across the kinds with invalid inputs and headers
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenants <q1_tenants>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenants <q2_tenants>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | q1_tenants | q2_tenants | kind | query | limit | returned_fields | first_count | response_code | reponse_type | response_message | errors |
- | "tenant1","common" | "tenant2","common" | "*:testcursor<timestamp>:well:1.0.0" | None | 1 | All | 1 | 401 | "Access denied" | "The user is not authorized to perform this action" | "" |
-
- @ignore
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- And I limit the count of returned results to <limit>
- And I send request to tenants <q1_tenants>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenants <q2_tenants>
- Then I should get in response <final_count> records
-
- Examples:
- | q1_tenants | q2_tenants | kind | query | limit | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | first_count | final_count |
- | "tenant1","common" | "tenant1","common" | "*:testcursor<timestamp>:well:1.0.0" | None | None | "data.Location" | 45 | -100 | 0 | 0 | 4 | 0 |
- | "tenant1","common" | "tenant1","common" | "tenant1:testcursor<timestamp>:well:1.0.0" | None | None | "data.Location" | 45 | -100 | 0 | 0 | 2 | 0 |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/querybycursor/singlecluster/QueryByCursor.feature b/testing/indexer-test-core/src/main/resources/features/querybycursor/singlecluster/QueryByCursor.feature
deleted file mode 100644
index 83a5171560414c2dcbbd5f5df801993aa1b2d887..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/querybycursor/singlecluster/QueryByCursor.feature
+++ /dev/null
@@ -1,92 +0,0 @@
-Feature: Search recursively on cursor with different queries
- To allow a user to find his data quickly, search should offer multiple ways to search data and iterate over all the results.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testcursor<timestamp>:well:1.0.0 | tenant1-testcursor<timestamp>-well-1.0.0 | records_1 | records_1 | data.default.viewers@opendes | data.default.owners@opendes |
- | tenant1:testcursor<timestamp>:well:2.0.0 | tenant1-testcursor<timestamp>-well-2.0.0 | records_2 | records_2 | data.default.viewers@opendes | data.default.testowners@opendes |
-
- Scenario Outline: Search recursively page by page data across the kinds
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenant <q1_tenant>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenant <q2_tenant>
- Then I should get in response <final_count> records
-
- Examples:
- | q1_tenant | q2_tenant | kind | query | limit | returned_fields | first_count | final_count |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:*:*" | None | 4 | All | 4 | 2 |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:*:*" | None | None | All | 6 | 0 |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:*:*" | "TX OR TEXAS OR FRANCE" | 1 | All | 1 | 1 |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:*:*" | "XdQQ6GCSNSBLTESTFAIL" | 1 | All | 0 | 0 |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:*:*" | "\"OFFICE2\" \| OFFICE3 \| OFFICE5" | 1 | All | 1 | 1 |
-
- Scenario Outline: Search recursively page by page data across the kinds with invalid inputs
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set an invalid cursor
- And I send request to tenant <tenant>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | tenant | kind | query | limit | response_code | reponse_type | response_message | errors |
- | "tenant1" | "tenant1:testcursor<timestamp>:well:1.0.0" | None | None | 400 | "Can't find the given cursor" | "The given cursor is invalid or expired" | "" |
- | "tenant1" | "*:*:*" | None | 0 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "Not a valid record kind. Found: *:*:*" |
- | "tenant1" | "tenant1:testcursor<timestamp>:well:1.0.0" | None | -1 | 400 | "Bad Request" | "Invalid parameters were given on search request" | "'limit' must be equal or greater than 0" |
-
- Scenario Outline: Search recursively page by page data across the kinds with invalid inputs and headers
- When I send <query> with <kind>
- And I limit the count of returned results to <limit>
- And I set the fields I want in response as <returned_fields>
- And I send request to tenant <q1_tenant>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenant <q2_tenant>
- Then I should get <response_code> response with reason: <reponse_type>, message: <response_message> and errors: <errors>
-
- Examples:
- | q1_tenant | q2_tenant | kind | query | limit | returned_fields | first_count | response_code | reponse_type | response_message | errors |
- | "tenant1" | "tenant2" | "tenant1:testcursor<timestamp>:well:1.0.0" | None | 1 | All | 1 | 401 | "Access denied" | "The user is not authorized to perform this action" | "" |
-
- Scenario Outline: Search data across the kinds with bounding box inputs
- When I send <query> with <kind>
- And I apply geographical query on field <field>
- And define bounding box with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>)
- And I limit the count of returned results to <limit>
- And I send request to tenant <q1_tenant>
- Then I should get in response <first_count> records along with a cursor
- And I send request to tenant <q2_tenant>
- Then I should get in response <final_count> records
-
- Examples:
- | q1_tenant | q2_tenant | kind | query | limit | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | first_count | final_count |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:well:1.0.0" | None | None | "data.Location" | 45 | -100 | 0 | 0 | 2 | 0 |
- | "tenant1" | "tenant1" | "tenant1:testcursor<timestamp>:well:1.0.0" | "OFFICE4" | 1 | "data.Location" | 45 | -110 | 0 | 0 | 1 | 0 |
-
- Scenario Outline: Search data and sort the results with the given sort fields and order
- When I send <query> with <kind>
- And I want the results sorted by <sort>
- Then I should get records in right order first record id: <first_record_id>, last record id: <last_record_id>
- Examples:
- | kind | query | sort | first_record_id | last_record_id |
- | "tenant1:testcursor<timestamp>:well:*" | None | {"field":["id"],"order":["ASC"]} | "test:well:1.0.0:1" | "test:well:2.0.0:3" |
- | "tenant1:testcursor<timestamp>:well:*" | None | {"field":["id"],"order":["DESC"]} | "test:well:2.0.0:3" | "test:well:1.0.0:1" |
- | "tenant1:testcursor<timestamp>:well:*" | None | {"field":["namespace","data.Rank"],"order":["ASC","DESC"]} | "test:well:1.0.0:3" | "test:well:2.0.0:1" |
-
- Scenario Outline: Search data in a given kind with different searchAs modes
- When I send <query> with <kind>
- And I want to search as owner <is_owner>
- Then I should get in response <count> records
-
- Examples:
- | kind | query | is_owner | count |
- | "tenant1:testcursor<timestamp>:well:1.0.0" | None | true | 3 |
- | "tenant1:testcursor<timestamp>:well:1.0.0" | None | false | 3 |
- | "tenant1:testcursor<timestamp>:well:2.0.0" | None | true | 0 |
- | "tenant1:testcursor<timestamp>:well:2.0.0" | None | false | 3 |
- | "tenant1:testcursor<timestamp>:well:*" | None | false | 6 |
- | "tenant1:testcursor<timestamp>:well:*" | None | true | 3 |
- | "tenant1:testcursor<timestamp>:well:*" | "OFFICE4"| true | 1 |
- | "tenant1:testcursor<timestamp>:well:*" | None | None | 6 |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/smart/parser/SmartParser.feature b/testing/indexer-test-core/src/main/resources/features/smart/parser/SmartParser.feature
deleted file mode 100644
index 408319b40c2c570d8f72ff07ab4333545b0d5ace..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/smart/parser/SmartParser.feature
+++ /dev/null
@@ -1,8 +0,0 @@
-Feature: Smart Parser
- To allow a client to parse a smart search query into a full search query syntax.
-
- Scenario: Parse smart search input to query api input
- When I generate smart search input with "text" and "well"
- Then I get a response matching
-
-
diff --git a/testing/indexer-test-core/src/main/resources/features/smart/search/Smart.feature b/testing/indexer-test-core/src/main/resources/features/smart/search/Smart.feature
deleted file mode 100644
index 3da32045c661d617ce821555256471b6e0f06d71..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/smart/search/Smart.feature
+++ /dev/null
@@ -1,30 +0,0 @@
-Feature: Smart search
- To allow a client to get the available filters, find the possible values to use for the filter and perform a search based on a filter and value.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testquery<timestamp>:well:1.0.0 | tenant1-testquery<timestamp>-well-1.0.0 | records_1 | records_1 | data.search.integrationtest@tenant1 | data.search.integrationtest@tenant1 |
- | tenant1:testquery<timestamp>:well:2.0.0 | tenant1-testquery<timestamp>-well-2.0.0 | records_2 | records_2 | data.search.integrationtest@tenant1 | data.search.integrationtestowner@tenant1 |
- # TODO: Enable the test when ECE CCS is utilized, the test looks correct, but it does not correspond to the current stopgap specification
- @ignore
- Scenario Outline: Perform smart search based on query
- When I synchronize the values in cache
- When I list all filters on tenants <tenants>
- Then I get the list of all available filters
- Then I take the first filter and list values when query is <query>
- Then I search with the filter and values and <limit> and <offset> for matching response
-
- Examples:
- | tenants | query | limit | offset |
- | "common","tenant1" | "w" | 2 | 1 |
- @ignore
- Scenario Outline: Perform smart search based on filters and values
- When I list all filters on tenants <tenants>
- When I take the <filter> and value <value>
- Then I search response matching for offset <offset> and limit <limit>
-
- Examples:
- | tenants | limit | offset | filter | value |
- | "common","tenant1" | 1 | 0 | "text" | "S" |
- | "common","tenant1" | 1 | 0 | "Field,Operator" | "S" |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/updateindex/UpdateIndexMapping.feature b/testing/indexer-test-core/src/main/resources/features/updateindex/UpdateIndexMapping.feature
deleted file mode 100644
index 23271eea8cc6acf1f7b7685a1c6537371b022ee9..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/updateindex/UpdateIndexMapping.feature
+++ /dev/null
@@ -1,20 +0,0 @@
-Feature: Updating elastic index mapping
- This feature deals with updating Index mapping in Elastic Search.
-
- Background:
- Given the elastic search is initialized with the following data
- | kind | index | mappingFile | recordFile | viewerGroup | ownerGroup |
- | tenant1:testupdatemapping<timestamp>:well:1.0.0 | tenant1-testupdatemapping<timestamp>-well-1.0.0 | records_3 | records_3 | data.default.viewers@opendes | data.default.owners@opendes |
- | tenant1:testupdatemapping<timestamp>:well:2.0.0 | tenant1-testupdatemapping<timestamp>-well-2.0.0 | records_3 | records_3 | data.default.viewers@opendes | data.default.owners@opendes |
-
- Scenario Outline: Update indices to enable multifield indexing
- When I update <fieldName> in <indices> to enable multifield indexing
- And I send request to tenant <tenant>
- Then I should get <response_code> response
- Then I want to validate mapping by <indices> and <fieldName> and <type>
- When I send <query> with <kind>
- And I want to aggregate by <fieldName>
- Then I should get in response <count> records
- Examples:
- | tenant | fieldName | type | kind | indices | response_code | query | count |
- | "tenant1" | "Center" | "well" |"tenant1:testupdatemapping<timestamp>:*:*" | "tenant1-testupdatemapping<timestamp>-well-1.0.0" ,"tenant1-testupdatemapping<timestamp>-well-2.0.0" | 200 | None | 6 |
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/RunTest.java
deleted file mode 100644
index b04eb1b3079df08037ce32ada4f8f007c941e410..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/RunTest.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.opengroup.osdu.step_definitions.index.deleteschema;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-@RunWith(Cucumber.class)
-@CucumberOptions(
- features = "classpath:features/delete/Delete.feature",
- glue={"classpath:org.opengroup.osdu.step_definitions/index/deleteschema"},
- format = {"pretty", "junit:target/cucumber-reports/TEST-deleteschema.xml"})
-public class RunTest {
-}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/Steps.java
deleted file mode 100644
index 344d597375249c43a2db50cea58ec6177e1346d3..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/deleteschema/Steps.java
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.opengroup.osdu.step_definitions.index.deleteschema;
-
-import cucumber.api.Scenario;
-import cucumber.api.java.Before;
-import org.opengroup.osdu.common.DeleteSchemaSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
-import cucumber.api.DataTable;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-
-import java.util.List;
-
-public class Steps extends DeleteSchemaSteps {
-
- public Steps() {
- super(new GCPHTTPClient());
- }
-
- @Before
- public void before(Scenario scenario) {
- this.scenario = scenario;
- this.httpClient = new GCPHTTPClient();
- }
-
- @Given("^the elastic search is initialized with the following data$")
- public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
- super.the_elastic_search_is_initialized_with_the_following_data(dataTable);
- }
-
- @When("^I send a delete request with \"([^\"]*)\"$")
- public void i_send_a_delete_request_with(String kind) throws Throwable {
- super.i_send_a_delete_request_with(kind);
- }
-
- @Then("^the index should get delete and I should get (\\d+) response$")
- public void the_index_should_get_delete_and_I_should_get_response(int code) throws Throwable {
- super.the_index_should_get_delete_and_I_should_get_response(code);
- }
-
- @Then("^I should get ([^\"]*) response with reason: \"(.*?)\", message: \"(.*?)\" and errors: \"(.*?)\"$")
- public void i_should_get_response_with_reason_message_and_errors(List<Integer> codes, String type, String msg,
- String error) throws Throwable {
- super.i_should_get_response_with_reason_message_and_errors(codes, type, msg, error);
- }
-
-}
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/RunTest.java
deleted file mode 100644
index b60b4a1c581517f89f6e0b447738847cb2f75872..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/RunTest.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.opengroup.osdu.step_definitions.index.getschema;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-
-@RunWith(Cucumber.class)
-@CucumberOptions(
- features = "classpath:features/kindschema/KindSchema.feature",
- glue = {"classpath:org.opengroup.osdu.step_definitions/index/getschema"},
- format = {"pretty", "junit:target/cucumber-reports/TEST-getschema.xml"})
-public class RunTest {
-}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/Steps.java
deleted file mode 100644
index 349f6163f72cf6719526e011c210a8f55a0596a0..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/getschema/Steps.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package org.opengroup.osdu.step_definitions.index.getschema;
-
-import org.opengroup.osdu.common.GetSchemaSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
-import cucumber.api.Scenario;
-import cucumber.api.java.Before;
-import cucumber.api.DataTable;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-
-public class Steps extends GetSchemaSteps {
- public Steps() {
- super(new GCPHTTPClient());
- }
-
- @Before
- public void before(Scenario scenario) {
- this.scenario = scenario;
- this.httpClient = new GCPHTTPClient();
- }
-
- @Given("^the elastic search is initialized with the following data$")
- public void the_elastic_search_is_initialized_with_the_following_data(DataTable dataTable) throws Throwable {
- super.the_elastic_search_is_initialized_with_the_following_data(dataTable);
- }
-
- @When("^I send get schema request with \"([^\"]*)\"$")
- public void i_send_get_schema_request_with(String kind) throws Throwable {
- super.i_send_get_schema_request_with(kind);
- }
-
- @When("^I send request to tenant \"(.*?)\"$")
- public void i_send_request_to_tenant(String tenant) throws Throwable {
- super.i_send_request_to_tenant(tenant);
- }
-
- @Then("^I should get ([^\"]*) response with reason: \"(.*?)\", message: \"(.*?)\" and errors: \"(.*?)\"$")
- public void i_should_get_response_with_reason_message_and_errors(int responseCode, String type, String msg,
- String error) throws Throwable {
- super.i_should_get_response_with_reason_message_and_errors(responseCode, type, msg, error);
- }
-
- @Then("^I should get (\\d+) status with response \"(.*?)\"$")
- public void i_should_get_status_with_response(int statusCode, String response) throws Throwable {
- super.i_should_get_status_with_response(statusCode, response);
- }
-
-}
\ No newline at end of file
diff --git a/testing/maven/settings.xml b/testing/maven/settings.xml
index 601aa71ea3679d558de22b4e530c9dc2a9467f7f..2a633d9a30c1997609060dcc31667d3dc1b6774a 100644
--- a/testing/maven/settings.xml
+++ b/testing/maven/settings.xml
@@ -4,10 +4,10 @@
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
- <id>dev-azure-com-slb-des-ext-collaboration-os-core</id>
- <username>os-core</username>
+ <id>os-core</id>
+ <username>slb-des-ext-collaboration</username>
<!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 10/8/2019 -->
+ <!-- The generated token expires on or before 11/14/2019 -->
<password>${VSTS_FEED_TOKEN}</password>
</server>
</servers>