Skip to content
GitLab
Commit 894d67a6 authored by Aleh Shubko [EPAM]'s avatar Aleh Shubko [EPAM]
Browse files

Merge branch 'mongodb-batch_operation_and_update_spring_config' into 'master' 

Batch operation and update spring config

See merge request !3
parents 22b7192e 9fc83f68
Pipeline #36427 passed with stages
in 5 minutes and 7 seconds
Hide whitespace changes
Inline Side-by-side
README.md
View file @ 894d67a6
......@@ -14,12 +14,13 @@ List of services supported:
<dependency>
<groupId>org.opengroup.osdu.core.mongodb</groupId>
<artifactId>os-core-lib-mongodb</artifactId>
<version>0.1.0</version>
<version>0.1.1</version>
</dependency>
```
- Add following to your `application.properties` file:
```
osdu.mongodb.enabled=true
osdu.mongodb.enabled.storage=true # to enable storage helpers
osdu.mongodb.enabled.legal=true # to enable legal helpers
osdu.mongodb.database=[your_database_name]
osdu.mongodb.uri=[connection url to MongoDB Atlas cluster or any MongoDB instance/replica (no matter on-premice or cloud)]
```
......
pom.xml
View file @ 894d67a6
......@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu.core.mongodb</groupId>
<artifactId>os-core-lib-mongodb</artifactId>
<version>0.1.0</version>
<version>0.1.1</version>
<packaging>jar</packaging>
<name>os-core-lib-mongodb</name>
......
src/main/java/org/opengroup/osdu/core/mongodb/config/LegalContext.java 0 → 100644
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.config;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.opengroup.osdu.core.mongodb.helper.legal.LegalHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ConditionalOnProperty("osdu.mongodb.enabled.legal")
@Configuration
public class LegalContext {
@Autowired
@Bean
public LegalHelper legalHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new LegalHelper(basicMongoDBHelper);
}
}
src/main/java/org/opengroup/osdu/core/mongodb/config/MongoConfig.java
View file @ 894d67a6
......@@ -7,7 +7,7 @@ import org.opengroup.osdu.core.mongodb.config.converter.SqlDateWriteConverter;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
......@@ -25,11 +25,10 @@ import java.util.List;
/**
* Basic configuration for MongoDB beans.
* Conditional On properties is used to prevent auto scanning for services
* (so beans will only be created when needed).
* Will be imported from other services
*/
@ConditionalOnProperty("osdu.mongodb.enabled")
@Configuration
@ConditionalOnExpression("${osdu.mongodb.enabled.storage:false} or ${osdu.mongodb.enabled.legal:false}")
public class MongoConfig extends AbstractMongoConfiguration {
public static final String SQL_TIMESTAMP_REPLACER_NAME = "_sql_timestamp_replacer_";
......
src/main/java/org/opengroup/osdu/core/mongodb/config/StorageContext.java 0 → 100644
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.config;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.QueryHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.RecordHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.SchemaHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ConditionalOnProperty("osdu.mongodb.enabled.storage")
@Configuration
public class StorageContext {
@Autowired
@Bean
public SchemaHelper schemaHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new SchemaHelper(basicMongoDBHelper);
}
@Autowired
@Bean
public RecordHelper recordHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new RecordHelper(basicMongoDBHelper);
}
@Autowired
@Bean
public QueryHelper queryHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new QueryHelper(basicMongoDBHelper);
}
}
src/main/java/org/opengroup/osdu/core/mongodb/helper/BasicMongoDBHelper.java
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.helper;
import com.google.common.collect.Lists;
import com.mongodb.client.result.UpdateResult;
import io.jsonwebtoken.lang.Collections;
import org.opengroup.osdu.core.mongodb.entity.BasicMongoDBDoc;
import org.opengroup.osdu.core.mongodb.entity.QueryPageResult;
import org.opengroup.osdu.core.mongodb.exception.InvalidCursorException;
......@@ -10,8 +12,11 @@ import org.springframework.data.mongodb.core.index.Index;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import static org.springframework.data.domain.Sort.Direction.ASC;
......@@ -65,14 +70,32 @@ public class BasicMongoDBHelper {
* @param documentsToSave documents for saving.
*/
public <T> void save(Collection<T> documentsToSave) {
mongoOperations.insertAll(documentsToSave);
if (!CollectionUtils.isEmpty(documentsToSave)) {
mongoOperations.insertAll(documentsToSave);
}
}
/**
* Save documents into database in batches of batchSize
*
* @param documentsToSave documents for saving.
* @param batchSize maximum amount of documents that are saved in one operation
*/
public <T> void batchSave(List<T> documentsToSave, int batchSize) {
if (!CollectionUtils.isEmpty(documentsToSave)) {
if (batchSize < 1) {
throw new IllegalArgumentException(String.format("Batch size cannot be zero or less, got %d", batchSize));
}
Lists.partition(documentsToSave, batchSize).forEach(mongoOperations::insertAll);
}
}
/**
* Update an entity by the query
* @param query query to search by
*
* @param query query to search by
* @param update an update object
* @param clazz entity to update
* @param clazz entity to update
* @return update result
*/
public UpdateResult update(Query query, Update update, Class<?> clazz) {
......@@ -106,12 +129,13 @@ public class BasicMongoDBHelper {
/**
* Delete document from database by matching fieldName-value pair. Caution! Can delete multiple items.
*
* @param clazz class to determinate MongoDB collection.
* @param fieldName field name to be searched for.
* @param value objects with fieldName and this value will be deleted.
* @param clazz class to determinate MongoDB collection.
* @return was any entity deleted
*/
public <T> void delete(String fieldName, Object value, Class<T> clazz) {
mongoOperations.remove(Query.query(Criteria.where(fieldName).is(value)), clazz);
public <T> boolean delete(String fieldName, Object value, Class<T> clazz) {
return mongoOperations.remove(Query.query(Criteria.where(fieldName).is(value)), clazz).getDeletedCount() > 0;
}
/**
......@@ -134,7 +158,8 @@ public class BasicMongoDBHelper {
* @param limit maximum returned document count for current page.
* @return {@link QueryPageResult} found document list with cursor for next page (if page exists).
*/
public <T extends BasicMongoDBDoc> QueryPageResult<T> queryPage(Query searchQuery, String idFieldName, String cursorValue, Class<T> clazz, Integer limit) throws InvalidCursorException {
public <T extends BasicMongoDBDoc> QueryPageResult<T> queryPage(
Query searchQuery, String idFieldName, String cursorValue, Class<T> clazz, Integer limit) throws InvalidCursorException {
if (cursorValue != null) {
if (!mongoOperations.exists(Query.query(Criteria.where(idFieldName).is(cursorValue)), clazz)) {
......
src/main/java/org/opengroup/osdu/core/mongodb/helper/legal/LegalHelper.java
View file @ 894d67a6
......@@ -2,6 +2,7 @@ package org.opengroup.osdu.core.mongodb.helper.legal;
import com.mongodb.client.result.UpdateResult;
import org.apache.http.HttpStatus;
import org.jboss.logging.Logger;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.legal.LegalTag;
import org.opengroup.osdu.core.mongodb.entity.legal.LegalDoc;
......@@ -22,6 +23,7 @@ import static org.opengroup.osdu.core.mongodb.entity.DomainFields.TENANT;
public class LegalHelper {
private final Logger logger = Logger.getLogger(this.getClass());
private final BasicMongoDBHelper mongoDBHelper;
public LegalHelper(BasicMongoDBHelper mongoDBHelper) {
......@@ -69,11 +71,11 @@ public class LegalHelper {
*/
public Boolean delete(long legalTagId) {
try {
mongoDBHelper.delete(ID, String.valueOf(legalTagId), LegalDoc.class);
return true;
return mongoDBHelper.delete(ID, String.valueOf(legalTagId), LegalDoc.class);
} catch (Exception e) {
e.printStackTrace();
return false;
logger.error(e);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("Error deleting legal tag, id = %d", legalTagId),
e.getMessage());
}
}
......@@ -111,7 +113,7 @@ public class LegalHelper {
Query query = Query.query(Criteria.where(TENANT).is(tenant)).limit(count);
docs = mongoDBHelper.find(query, LegalDoc.class);
} catch (Exception e) {
e.printStackTrace();
logger.error(e);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error parsing results",
e.getMessage());
}
......
src/main/java/org/opengroup/osdu/core/mongodb/helper/storage/QueryHelper.java
View file @ 894d67a6
......@@ -7,7 +7,6 @@ import org.opengroup.osdu.core.mongodb.entity.storage.SchemaDoc;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
......@@ -19,7 +18,6 @@ import static org.opengroup.osdu.core.mongodb.entity.DomainFields.KIND;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.STATUS;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.RECORD_STATUS;
@Service
public class QueryHelper {
private static final int PAGE_SIZE = 1000;
......
src/main/java/org/opengroup/osdu/core/mongodb/helper/storage/RecordHelper.java
View file @ 894d67a6
......@@ -9,7 +9,6 @@ import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.index.Index;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.AbstractMap;
......@@ -27,9 +26,9 @@ import static org.opengroup.osdu.core.mongodb.entity.DomainFields.USER;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.ID;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.ASSOCIATION_RECORD_ID;
@Service
public class RecordHelper {
private static final int BATCH_SIZE = 100;
private final BasicMongoDBHelper mongoDBHelper;
......@@ -55,8 +54,8 @@ public class RecordHelper {
docs.add(doc);
});
mongoDBHelper.save(docs);
mongoDBHelper.save(associationList);
mongoDBHelper.batchSave(docs, BATCH_SIZE);
mongoDBHelper.batchSave(associationList, BATCH_SIZE);
}
public RecordMetadata get(String id) {
......
src/main/java/org/opengroup/osdu/core/mongodb/helper/storage/SchemaHelper.java
View file @ 894d67a6
......@@ -5,7 +5,6 @@ import org.opengroup.osdu.core.mongodb.entity.storage.SchemaDoc;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.index.Index;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
......@@ -13,7 +12,6 @@ import static org.opengroup.osdu.core.mongodb.entity.DomainFields.TENANT;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.USER;
import static org.opengroup.osdu.core.mongodb.entity.DomainFields.ID;
@Service
public class SchemaHelper {
private final BasicMongoDBHelper mongoDBHelper;
......
src/test/java/org/opengroup/osdu/core/mongodb/helper/BasicMongoDBHelperTests.java
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.helper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.mongodb.entity.BasicMongoDBDoc;
......@@ -11,10 +14,11 @@ import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.query.Query;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.*;
@RunWith(MockitoJUnitRunner.class)
public class BasicMongoDBHelperTests {
......@@ -22,8 +26,19 @@ public class BasicMongoDBHelperTests {
private final MongoOperations mongoOperations = Mockito.mock(MongoOperations.class);
private final BasicMongoDBHelper helper = new BasicMongoDBHelper(mongoOperations);
@Before
public void initMocks() {
reset(mongoOperations);
}
@After
public void verifyMocks() {
verifyNoMoreInteractions(mongoOperations);
}
@Test
public void testPaginationFirstPage() {
// given
BasicMongoDBDoc doc1 = Mockito.mock(BasicMongoDBDoc.class);
BasicMongoDBDoc doc2 = Mockito.mock(BasicMongoDBDoc.class);
BasicMongoDBDoc doc3 = Mockito.mock(BasicMongoDBDoc.class);
......@@ -36,6 +51,7 @@ public class BasicMongoDBHelperTests {
Query query = new Query();
//when
when(mongoOperations.find(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class)))
.thenReturn(page);
......@@ -43,18 +59,23 @@ public class BasicMongoDBHelperTests {
query, "id", null, BasicMongoDBDoc.class, 3
);
// then
Mockito.verify(mongoOperations).find(query, BasicMongoDBDoc.class);
assertThat(result.getResults()).containsAll(page);
assertThat(result.getCursor()).isEqualTo("3");
}
@Test
public void testPaginationLastPage() {
BasicMongoDBDoc doc1 = Mockito.mock(BasicMongoDBDoc.class);
// given
BasicMongoDBDoc<?> document = Mockito.mock(BasicMongoDBDoc.class);
List<BasicMongoDBDoc> page = new ArrayList<>();
page.add(doc1);
page.add(document);
Query query = new Query();
// when
when(mongoOperations.exists(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class)))
.thenReturn(true);
when(mongoOperations.find(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class)))
......@@ -64,24 +85,81 @@ public class BasicMongoDBHelperTests {
query, "id", "0", BasicMongoDBDoc.class, 3
);
// then
InOrder orderVerifier = Mockito.inOrder(mongoOperations);
orderVerifier.verify(mongoOperations).exists(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class));
orderVerifier.verify(mongoOperations).find(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class));
assertThat(result.getResults()).containsAll(page);
assertThat(result.getCursor()).isNull();
}
@Test(expected = InvalidCursorException.class)
public void testPaginationWithWrongCursor() {
when(mongoOperations.exists(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class)))
.thenReturn(false);
doReturn(false).when(mongoOperations).exists(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class));
try {
helper.queryPage(
new Query(), "id", "foo", BasicMongoDBDoc.class, 3
);
} finally {
Mockito.verify(mongoOperations).exists(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class));
}
}
helper.queryPage(
new Query(), "id", "foo", BasicMongoDBDoc.class, 3
);
@Test
public void testBatchSaving() {
// given
List<BasicMongoDBDoc<?>> docs = new ArrayList<>();
docs.add(Mockito.mock(BasicMongoDBDoc.class));
docs.add(Mockito.mock(BasicMongoDBDoc.class));
docs.add(Mockito.mock(BasicMongoDBDoc.class));
Collection<BasicMongoDBDoc<?>> req1 = new ArrayList<>();
Collection<BasicMongoDBDoc<?>> req2 = new ArrayList<>();
req1.add(docs.get(0));
req1.add(docs.get(1));
req2.add(docs.get(2));
// when
helper.batchSave(docs, 2);
// then
InOrder orderVerifier = Mockito.inOrder(mongoOperations);
orderVerifier.verify(mongoOperations).insertAll(req1);
orderVerifier.verify(mongoOperations).insertAll(req2);
}
@Test
public void testBatchSavingInSizeOfBatch() {
// given
List<BasicMongoDBDoc<?>> docs = new ArrayList<>();
docs.add(Mockito.mock(BasicMongoDBDoc.class));
docs.add(Mockito.mock(BasicMongoDBDoc.class));
docs.add(Mockito.mock(BasicMongoDBDoc.class));
// when
helper.batchSave(docs, 3);
// then
Mockito.verify(mongoOperations).insertAll(docs);
Mockito.verifyNoMoreInteractions(mongoOperations);
}
@Test(expected = IllegalArgumentException.class)
public void testBatchSizeError() {
List<BasicMongoDBDoc<?>> docs = new ArrayList<>();
docs.add(Mockito.mock(BasicMongoDBDoc.class));
helper.batchSave(docs, 0);
}
@Test
public void testEmptyPage() {
// given
List<BasicMongoDBDoc> emptyList = new ArrayList<>();
// when
when(mongoOperations.find(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class)))
.thenReturn(emptyList);
......@@ -89,6 +167,9 @@ public class BasicMongoDBHelperTests {
new Query(), "id", null, BasicMongoDBDoc.class, 3
);
// then
Mockito.verify(mongoOperations).find(Mockito.any(), Mockito.eq(BasicMongoDBDoc.class));
assertThat(result.getResults()).isNotNull();
assertThat(result.getCursor()).isNull();
assertThat(result.getResults()).isEmpty();
......
src/test/java/org/opengroup/osdu/core/mongodb/integration/config/LegalTestConfig.java
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.integration.config;
import org.opengroup.osdu.core.mongodb.config.LegalContext;
import org.opengroup.osdu.core.mongodb.config.MongoConfig;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.opengroup.osdu.core.mongodb.helper.legal.LegalHelper;
......@@ -8,15 +9,8 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.test.context.ContextConfiguration;
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
@Import({MongoConfig.class})
@Import({MongoConfig.class, LegalContext.class})
@ContextConfiguration
public class LegalTestConfig {
@Autowired
@Bean
public LegalHelper legalHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new LegalHelper(basicMongoDBHelper);
}
}
src/test/java/org/opengroup/osdu/core/mongodb/integration/config/StorageTestConfig.java
View file @ 894d67a6
package org.opengroup.osdu.core.mongodb.integration.config;
import org.opengroup.osdu.core.mongodb.config.MongoConfig;
import org.opengroup.osdu.core.mongodb.helper.BasicMongoDBHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.QueryHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.RecordHelper;
import org.opengroup.osdu.core.mongodb.helper.storage.SchemaHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.opengroup.osdu.core.mongodb.config.StorageContext;
import org.springframework.context.annotation.Import;
import org.springframework.test.context.ContextConfiguration;
@Import({MongoConfig.class})
@Import({MongoConfig.class, StorageContext.class})
@ContextConfiguration
public class StorageTestConfig {
@Autowired
@Bean
public SchemaHelper schemaHelper( BasicMongoDBHelper basicMongoDBHelper) {
return new SchemaHelper(basicMongoDBHelper);
}
@Autowired
@Bean
public RecordHelper recordHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new RecordHelper(basicMongoDBHelper);
}
@Autowired
@Bean
public QueryHelper queryRepoHelper(BasicMongoDBHelper basicMongoDBHelper) {
return new QueryHelper(basicMongoDBHelper);
}
}
src/test/java/org/opengroup/osdu/core/mongodb/integration/legal/LegalTests.java
View file @ 894d67a6
......@@ -5,6 +5,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.opengroup.osdu.core.common.model.legal.LegalTag;
import org.opengroup.osdu.core.mongodb.config.LegalContext;
import org.opengroup.osdu.core.mongodb.config.MongoConfig;
import org.opengroup.osdu.core.mongodb.integration.config.LegalTestConfig;
import org.opengroup.osdu.core.mongodb.integration.config.MongoEmbeddedConfig;
import org.opengroup.osdu.core.mongodb.entity.legal.LegalDoc;
......@@ -49,10 +51,11 @@ public class LegalTests {
public void testCreate() {
LegalTag tag = LegalTagGenerator.get("name-1");
legalHelper.create(tag, TENANT);
Long id = legalHelper.create(tag, TENANT);
LegalDoc doc = mongoTemplate.findById(String.valueOf(tag.getId()), LegalDoc.class);
assertThat(doc).isNotNull();
assertThat(id).isEqualTo(tag.getId());
}
@Test
......@@ -91,15 +94,17 @@ public class LegalTests {
@Test
public void testUpdate() {
// given
LegalTag tag = insertDoc("name-4");
String newDescription = tag.getDescription() + " - NEW";
long id = tag.getId();
// when
tag.setDescription(newDescription);
legalHelper.update(tag, TENANT);
LegalDoc doc = mongoTemplate.findById(String.valueOf(id), LegalDoc.class);
// then
assertThat(doc).isNotNull();
assertThat(doc.getData()).isNotNull();
assertThat(doc.getTenant()).isEqualTo(TENANT);
......