Commit d601dc28 authored by Yauheni Lesnikau's avatar Yauheni Lesnikau Committed by Smitha Manjunath
Browse files

bulk soft delete record api

parent a411ab7b
......@@ -105,6 +105,13 @@ public class RecordApi {
return new ResponseEntity<Void>(HttpStatus.NO_CONTENT);
}
@PostMapping(value = "/delete", consumes = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')")
public ResponseEntity<Void> bulkDeleteRecords(@RequestBody @NotEmpty @Size(max = 500, message = ValidationDoc.RECORDS_MAX) List<String> recordIs) {
this.recordService.bulkDeleteRecords(recordIs, this.headers.getUserEmail());
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
}
@GetMapping(value = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')")
public ResponseEntity<String> getLatestRecordVersion(
......
package org.opengroup.osdu.storage.exception;
import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
public class DeleteRecordsException extends RuntimeException {
private final List<Pair<String, String>> notDeletedRecords;
public DeleteRecordsException(List<Pair<String, String>> notDeletedRecords) {
this.notDeletedRecords = notDeletedRecords;
}
public List<Pair<String, String>> getNotDeletedRecords() {
return notDeletedRecords;
}
}
......@@ -14,9 +14,13 @@
package org.opengroup.osdu.storage.service;
import java.util.List;
public interface RecordService {
void purgeRecord(String recordId);
void deleteRecord(String recordId, String user);
void bulkDeleteRecords(List<String> records, String user);
}
\ No newline at end of file
......@@ -15,8 +15,10 @@
package org.opengroup.osdu.storage.service;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsAndCacheService;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
......@@ -25,17 +27,23 @@ import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
import org.opengroup.osdu.core.common.model.storage.RecordState;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.storage.exception.DeleteRecordsException;
import org.opengroup.osdu.storage.logging.StorageAuditLogger;
import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage;
import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
import org.opengroup.osdu.storage.util.api.RecordUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
@Service
public class RecordServiceImpl implements RecordService {
......@@ -61,6 +69,9 @@ public class RecordServiceImpl implements RecordService {
@Autowired
private DataAuthorizationService dataAuthorizationService;
@Autowired
private RecordUtil recordUtil;
@Override
public void purgeRecord(String recordId) {
......@@ -116,6 +127,47 @@ public class RecordServiceImpl implements RecordService {
this.pubSubClient.publishMessage(this.headers, pubSubInfo);
}
@Override
public void bulkDeleteRecords(List<String> records, String user) {
recordUtil.validateRecordIds(records);
List<Pair<String, String>> notDeletedRecords = new ArrayList<>();
List<RecordMetadata> recordsMetadata = getRecordsMetadata(records, notDeletedRecords);
this.validateAccess(recordsMetadata, notDeletedRecords);
Date modifyTime = new Date();
recordsMetadata.forEach(recordMetadata -> {
recordMetadata.setStatus(RecordState.deleted);
recordMetadata.setModifyTime(modifyTime.getTime());
recordMetadata.setModifyUser(user);
}
);
if (notDeletedRecords.isEmpty()) {
this.recordRepository.createOrUpdate(recordsMetadata);
this.auditLogger.deleteRecordSuccess(records);
publishDeletedRecords(recordsMetadata);
} else {
List<String> deletedRecords = new ArrayList<>(records);
List<String> notDeletedRecordIds = notDeletedRecords.stream()
.map(Pair::getKey)
.collect(toList());
deletedRecords.removeAll(notDeletedRecordIds);
if(!deletedRecords.isEmpty()) {
this.recordRepository.createOrUpdate(recordsMetadata);
this.auditLogger.deleteRecordSuccess(deletedRecords);
publishDeletedRecords(recordsMetadata);
}
throw new DeleteRecordsException(notDeletedRecords);
}
}
private void publishDeletedRecords(List<RecordMetadata> records) {
List<PubSubInfo> messages = records.stream()
.map(recordMetadata -> new PubSubInfo(recordMetadata.getId(), recordMetadata.getKind(), OperationType.delete))
.collect(Collectors.toList());
pubSubClient.publishMessage(headers, messages.toArray(new PubSubInfo[messages.size()]));
}
private RecordMetadata getRecordMetadata(String recordId, boolean isPurgeRequest) {
String tenantName = tenant.getName();
......@@ -137,10 +189,36 @@ public class RecordServiceImpl implements RecordService {
return record;
}
private List<RecordMetadata> getRecordsMetadata(List<String> recordIds, List<Pair<String, String>> notDeletedRecords) {
Map<String, RecordMetadata> result = this.recordRepository.get(recordIds);
recordIds.stream()
.filter(recordId -> result.get(recordId) == null)
.forEach(recordId -> {
String msg = String.format("Record with id '%s' not found", recordId);
notDeletedRecords.add(new ImmutablePair<>(recordId, msg));
auditLogger.deleteRecordFail(singletonList(msg));
});
return result.entrySet().stream().map(Map.Entry::getValue).collect(toList());
}
private void validateDeleteAllowed(RecordMetadata recordMetadata) {
if (!this.dataAuthorizationService.hasAccess(recordMetadata, OperationType.delete)) {
this.auditLogger.deleteRecordFail(singletonList(recordMetadata.getId()));
throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", "The user is not authorized to perform this action");
}
}
private void validateAccess(List<RecordMetadata> recordsMetadata, List<Pair<String, String>> notDeletedRecords) {
new ArrayList<>(recordsMetadata).forEach(recordMetadata -> {
if (!this.dataAuthorizationService.hasAccess(recordMetadata, OperationType.delete)) {
String msg = String
.format("The user is not authorized to perform delete record with id %s", recordMetadata.getId());
this.auditLogger.deleteRecordFail(singletonList(msg));
notDeletedRecords.add(new ImmutablePair<>(recordMetadata.getId(), msg));
recordsMetadata.remove(recordMetadata);
}
});
}
}
\ No newline at end of file
......@@ -14,14 +14,21 @@
package org.opengroup.osdu.storage.util;
import static org.apache.http.HttpStatus.SC_MULTI_STATUS;
import javax.validation.ValidationException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import javassist.NotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.storage.exception.DeleteRecordsException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
......@@ -92,6 +99,21 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
}
}
@ExceptionHandler(DeleteRecordsException.class)
protected ResponseEntity<Object> handleDeleteRecordsException(DeleteRecordsException e) {
JsonArray responseArray = new JsonArray();
e.getNotDeletedRecords().stream()
.map(pair -> {
JsonObject jsonObject = new JsonObject();
jsonObject.add("notDeletedRecordId", new JsonPrimitive(pair.getKey()));
jsonObject.add("message", new JsonPrimitive(pair.getValue()));
return jsonObject;
})
.forEach(responseArray::add);
return ResponseEntity.status(SC_MULTI_STATUS).body(responseArray.toString());
}
@Override
@NonNull
protected ResponseEntity<Object> handleHttpRequestMethodNotSupported(@NonNull HttpRequestMethodNotSupportedException e,
......
......@@ -14,6 +14,7 @@
package org.opengroup.osdu.storage.api;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
......@@ -86,7 +87,7 @@ public class RecordApiTest {
@Test
public void should_returnsHttp201_when_creatingOrUpdatingRecordsSuccessfully() {
TransferInfo transfer = new TransferInfo();
transfer.setSkippedRecords(Collections.singletonList("ID1"));
transfer.setSkippedRecords(singletonList("ID1"));
transfer.setVersion(System.currentTimeMillis() * 1000L + (new Random()).nextInt(1000) + 1);
Record r1 = new Record();
......@@ -189,6 +190,13 @@ public class RecordApiTest {
assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatusCodeValue());
}
@Test
public void should_returnHttp204_when_bulkDeleteRecordsSuccessfully() {
ResponseEntity response = this.sut.bulkDeleteRecords(singletonList(RECORD_ID));
assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatusCodeValue());
}
@Test
public void should_allowAccessToCreateOrUpdateRecords_when_userBelongsToCreatorOrAdminGroups() throws Exception {
......@@ -233,6 +241,16 @@ public class RecordApiTest {
assertTrue(annotation.value().contains(StorageRole.ADMIN));
}
@Test
public void should_allowAccessToBulkDeleteRecords_when_userBelongsToCreatorOrAdminGroups() throws Exception {
Method method = this.sut.getClass().getMethod("deleteRecord", String.class);
PreAuthorize annotation = method.getAnnotation(PreAuthorize.class);
assertFalse(annotation.value().contains(StorageRole.VIEWER));
assertTrue(annotation.value().contains(StorageRole.CREATOR));
assertTrue(annotation.value().contains(StorageRole.ADMIN));
}
@Test
public void should_allowAccessToGetLatestVersionOfRecord_when_userBelongsToViewerCreatorOrAdminGroups()
throws Exception {
......
......@@ -14,8 +14,12 @@
package org.opengroup.osdu.storage.service;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
......@@ -31,6 +35,7 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.storage.*;
import org.opengroup.osdu.core.common.storage.IPersistenceService;
import org.opengroup.osdu.storage.exception.DeleteRecordsException;
import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage;
import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
......@@ -48,6 +53,7 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.storage.logging.StorageAuditLogger;
import org.opengroup.osdu.core.common.storage.PersistenceHelper;
import org.opengroup.osdu.storage.util.api.RecordUtil;
@RunWith(MockitoJUnitRunner.class)
public class RecordServiceImplTest {
......@@ -55,6 +61,14 @@ public class RecordServiceImplTest {
private static final String RECORD_ID = "tenant1:record:anyId";
private static final String TENANT_NAME = "TENANT1";
private static final String RECORD_ID_1 = "tenant1:record1:version";
private static final String USER_NAME = "testUserName";
private static final String KIND = "testKind";
private static final String[] OWNERS = new String[]{"owner1@slb.com", "owner2@slb.com"};
private static final String[] VIEWERS = new String[]{"viewer1@slb.com", "viewer2@slb.com"};
@Mock
private IRecordsMetadataRepository recordRepository;
......@@ -79,6 +93,9 @@ public class RecordServiceImplTest {
@Mock
private ITenantFactory tenantFactory;
@Mock
private RecordUtil recordUtil;
@InjectMocks
private RecordServiceImpl sut;
......@@ -347,4 +364,145 @@ public class RecordServiceImplTest {
fail("Should not get different exception");
}
}
@Test
public void shouldDeleteRecords_successfully() {
RecordMetadata record = buildRecordMetadata();
Map<String, RecordMetadata> expectedRecordMetadataMap = new HashMap<String, RecordMetadata>(){{
put(RECORD_ID, record);
}};
when(recordRepository.get(singletonList(RECORD_ID))).thenReturn(expectedRecordMetadataMap);
when(dataAuthorizationService.hasAccess(record, OperationType.delete)).thenReturn(true);
sut.bulkDeleteRecords(singletonList(RECORD_ID), USER_NAME);
verify(recordRepository, times(1)).get(singletonList(RECORD_ID));
verify(dataAuthorizationService, only()).hasAccess(record, OperationType.delete);
verify(recordRepository, times(1)).createOrUpdate(singletonList(record));
verify(auditLogger, only()).deleteRecordSuccess(singletonList(RECORD_ID));
verifyPubSubPublished();
assertEquals(RecordState.deleted, record.getStatus());
assertEquals(USER_NAME, record.getModifyUser());
assertNotNull(record.getModifyTime());
assertTrue(record.getModifyTime() != 0);
}
@Test
public void shouldThrowDeleteRecordsException_when_tryingToDeleteRecordsWhichUserDoesNotHaveAccessTo() {
RecordMetadata record = buildRecordMetadata();
Map<String, RecordMetadata> expectedRecordMetadataMap = new HashMap<String, RecordMetadata>(){{
put(RECORD_ID, record);
}};
when(recordRepository.get(singletonList(RECORD_ID))).thenReturn(expectedRecordMetadataMap);
when(dataAuthorizationService.hasAccess(record, OperationType.delete)).thenReturn(false);
try {
sut.bulkDeleteRecords(singletonList(RECORD_ID), USER_NAME);
fail("Should not succeed!");
} catch (DeleteRecordsException e) {
String errorMsg = String
.format("The user is not authorized to perform delete record with id %s", RECORD_ID);
verify(recordRepository, times(1)).get(singletonList(RECORD_ID));
verify(dataAuthorizationService, only()).hasAccess(record, OperationType.delete);
verify(recordRepository, never()).createOrUpdate(any());
verify(auditLogger, only()).deleteRecordFail(singletonList(errorMsg));
verifyZeroInteractions(pubSubClient);
assertEquals(1, e.getNotDeletedRecords().size());
assertEquals(RECORD_ID, e.getNotDeletedRecords().get(0).getKey());
assertEquals(errorMsg, e.getNotDeletedRecords().get(0).getValue());
assertEquals(RecordState.active, record.getStatus());
assertNull(record.getModifyUser());
} catch (Exception e) {
fail("Should not get different exception");
}
}
@Test
public void shouldThrowDeleteRecordsException_when_tryingToDeleteRecordsWhenRecordNotFound() {
RecordMetadata record = buildRecordMetadata();
Map<String, RecordMetadata> expectedRecordMetadataMap = new HashMap<String, RecordMetadata>(){{
put(RECORD_ID, record);
}};
when(recordRepository.get(asList(RECORD_ID, RECORD_ID_1))).thenReturn(expectedRecordMetadataMap);
when(dataAuthorizationService.hasAccess(record, OperationType.delete)).thenReturn(true);
try {
sut.bulkDeleteRecords(asList(RECORD_ID, RECORD_ID_1), USER_NAME);
fail("Should not succeed!");
} catch (DeleteRecordsException e) {
String expectedErrorMessage = "Record with id '" + RECORD_ID_1 + "' not found";
verify(recordRepository, times(1)).get(asList(RECORD_ID, RECORD_ID_1));
verify(dataAuthorizationService, only()).hasAccess(record, OperationType.delete);
verify(recordRepository, times(1)).createOrUpdate(singletonList(record));
verify(auditLogger, times(1)).deleteRecordSuccess(singletonList(RECORD_ID));
verify(auditLogger, times(1)).deleteRecordFail(singletonList(expectedErrorMessage));
verifyPubSubPublished();
assertEquals(RecordState.deleted, record.getStatus());
assertEquals(USER_NAME, record.getModifyUser());
assertNotNull(record.getModifyTime());
assertEquals(1, e.getNotDeletedRecords().size());
assertEquals(RECORD_ID_1, e.getNotDeletedRecords().get(0).getKey());
assertEquals(expectedErrorMessage, e.getNotDeletedRecords().get(0).getValue());
} catch (Exception e) {
fail("Should not get different exception");
}
}
@Test
public void shouldThrowAppException_when_tryingToDeleteRecordsForInvalidIds() {
String errorMsg = String.format("The record '%s' does not follow the naming convention: the first id component must be '%s'",
RECORD_ID, TENANT_NAME);
try {
doThrow(new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid record id", errorMsg))
.when(recordUtil).validateRecordIds(singletonList(RECORD_ID));
sut.bulkDeleteRecords(asList(RECORD_ID), USER_NAME);
fail("Should not succeed!");
} catch (AppException e) {
assertEquals(HttpStatus.SC_BAD_REQUEST, e.getError().getCode());
assertEquals("Invalid record id", e.getError().getReason());
assertEquals(errorMsg, e.getError().getMessage());
verifyZeroInteractions(recordRepository, entitlementsAndCacheService, auditLogger,pubSubClient);
} catch (Exception e) {
fail("Should not get different exception");
}
}
private void verifyPubSubPublished() {
ArgumentCaptor<PubSubInfo> pubsubMessageCaptor = ArgumentCaptor.forClass(PubSubInfo.class);
verify(this.pubSubClient).publishMessage(eq(this.headers), pubsubMessageCaptor.capture());
PubSubInfo capturedMessage = pubsubMessageCaptor.getValue();
assertEquals(RECORD_ID, capturedMessage.getId());
assertEquals(KIND, capturedMessage.getKind());
assertEquals(OperationType.delete, capturedMessage.getOp());
}
private RecordMetadata buildRecordMetadata() {
Acl acl = new Acl();
acl.setViewers(VIEWERS);
acl.setOwners(OWNERS);
RecordMetadata record = new RecordMetadata();
record.setKind(KIND);
record.setAcl(acl);
record.setId(RECORD_ID);
record.setStatus(RecordState.active);
record.setGcsVersionPaths(asList("path/1", "path/2", "path/3"));
return record;
}
}
\ No newline at end of file
package org.opengroup.osdu.storage.records;
import org.junit.After;
import org.junit.Before;
import org.opengroup.osdu.storage.util.AzureTestUtils;
public class TestLogicalBatchRecordsDelete extends LogicalBatchRecordsDeleteTests {
private static final AzureTestUtils azureTestUtils = new AzureTestUtils();
@Before
@Override
public void setup() throws Exception {
this.testUtils = new AzureTestUtils();
super.setup(azureTestUtils.getToken());
}
@After
@Override
public void tearDown() throws Exception {
this.testUtils = null;
super.tearDown(azureTestUtils.getToken());
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.records;
import com.google.common.collect.Lists;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.http.HttpStatus;
import org.junit.Test;
import org.opengroup.osdu.storage.util.*;
import java.util.List;
import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.apache.http.HttpStatus.SC_MULTI_STATUS;
import static org.apache.http.HttpStatus.SC_NOT_FOUND;
import static org.junit.Assert.assertEquals;
public abstract class LogicalBatchRecordsDeleteTests extends TestBase {
protected static final long NOW = System.currentTimeMillis();
protected static final String KIND = TenantUtils.getTenantName() + ":delete:inttest:1.0." + NOW;
protected static final String LEGAL_TAG = LegalTagUtils.createRandomName();
protected static final String RECORD_ID_1 = TenantUtils.getTenantName() + ":testint:" + NOW;
protected static final String RECORD_ID_2 = TenantUtils.getTenantName() + ":testint:" + NOW;
private static final String NOT_EXISTED_RECORD_ID = TenantUtils.getFirstTenantName() + ":notexisted:" + NOW;
@Test
public void should_deleteRecordsLogically_successfully() throws Exception {
String requestBody = String.format("[\"%s\",\"%s\"]", RECORD_ID_1, RECORD_ID_2);
ClientResponse response = TestUtils.send("records/delete", "POST",
HeaderUtils.getHeaders(TenantUtils.getTenantName(), testUtils.getToken()), requestBody, EMPTY);
assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatus());
response = TestUtils.send("records/" + RECORD_ID_1, "GET",
HeaderUtils.getHeaders(TenantUtils.getTenantName(), testUtils.getToken()), "", "");
assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatus());
response = TestUtils.send("records/" + RECORD_ID_2, "GET",
HeaderUtils.getHeaders(TenantUtils.getTenantName(), testUtils.getToken()), "", "");
assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatus());
}
@Test
public void should_deleteRecordsLogically_withPartialSuccess_whenOneRecordNotFound() throws Exception {