Skip to content
Snippets Groups Projects
Commit 4fbadde8 authored by VidyaDharani Lokam's avatar VidyaDharani Lokam
Browse files

Merge branch 'az/vl-cleanup-deadcode' into 'master' 

[MS-44412] Remove unused GCS Array Validation

See merge request !923
parents dcf855eb 08f97f39
No related branches found
No related tags found
1 merge request!923[MS-44412] Remove unused GCS Array Validation
Pipeline #281904 failed
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: publish
Hide whitespace changes
Inline Side-by-side
NOTICE
+ 1
− 1
View file @ 4fbadde8
......@@ -68,7 +68,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
- Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- Cloud Key Management Service (KMS) API v1-rev20240801-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- Cloud Storage JSON API v1-rev20240809-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections)
- Commons Digester (from http://commons.apache.org/digester/)
......
storage-core/src/main/java/org/opengroup/osdu/storage/di/GcsVersionPathLimitationConfig.java deleted 100644 → 0
+ 0
− 17
View file @ dcf855eb
package org.opengroup.osdu.storage.di;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Getter
@Setter
@Configuration
@ConfigurationProperties(prefix = "gcs.version.paths")
public class GcsVersionPathLimitationConfig {
private int maxLimit = 2000;
private String featureName = "enforce_gcsVersionPathsLimit_enabled";
}
storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java
+ 0
− 22
View file @ 4fbadde8
......@@ -30,7 +30,6 @@ import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.*;
import org.opengroup.osdu.core.common.model.storage.validation.ValidationDoc;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.storage.di.GcsVersionPathLimitationConfig;
import org.opengroup.osdu.core.common.util.CollaborationContextUtil;
import org.opengroup.osdu.storage.logging.StorageAuditLogger;
import org.opengroup.osdu.storage.opa.model.OpaError;
......@@ -90,10 +89,6 @@ public class IngestionServiceImpl implements IngestionService {
@Autowired
RecordBlocks recordBlocks;
@Autowired
GcsVersionPathLimitationConfig gcsVersionPathLimitationConfig;
@Override
public TransferInfo createUpdateRecords(boolean skipDupes, List<Record> inputRecords, String user, Optional<CollaborationContext> collaborationContext) {
this.validateKindFormat(inputRecords);
......@@ -104,9 +99,6 @@ public class IngestionServiceImpl implements IngestionService {
List<RecordProcessing> recordsToProcess = this.getRecordsForProcessing(skipDupes, inputRecords, transfer, collaborationContext);
if(featureFlag.isFeatureEnabled(gcsVersionPathLimitationConfig.getFeatureName())){
this.applyGcsVersionPathsLimit(recordsToProcess, transfer);
}
this.sendRecordsForProcessing(recordsToProcess, transfer, collaborationContext);
return transfer;
}
......@@ -455,18 +447,4 @@ public class IngestionServiceImpl implements IngestionService {
}
}
private void applyGcsVersionPathsLimit(List<RecordProcessing> recordsToProcess, TransferInfo transferInfo) {
List<RecordProcessing> skippedRecords = new ArrayList<>();
int maxLimit = gcsVersionPathLimitationConfig.getMaxLimit();
for (RecordProcessing recordProcessing : recordsToProcess) {
int versionPathsSize = recordProcessing.getRecordMetadata().getGcsVersionPaths().size();
if (versionPathsSize > maxLimit) {
logger.warning(String.format("Record %s has %s versions which is higher than the allowed limit of: %s. Record will not be ingested", recordProcessing.getRecordMetadata().getId(), versionPathsSize, maxLimit));
transferInfo.getSkippedRecords().add(recordProcessing.getRecordMetadata().getId());
skippedRecords.add(recordProcessing);
}
}
recordsToProcess.removeAll(skippedRecords);
}
}
storage-core/src/test/java/org/opengroup/osdu/storage/service/IngestionServiceImplTest.java
+ 0
− 99
View file @ 4fbadde8
......@@ -45,7 +45,6 @@ import org.opengroup.osdu.core.common.model.storage.*;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.partition.PartitionException;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.storage.di.GcsVersionPathLimitationConfig;
import org.opengroup.osdu.storage.logging.StorageAuditLogger;
import org.opengroup.osdu.storage.opa.model.OpaError;
import org.opengroup.osdu.storage.opa.model.ValidationOutputRecord;
......@@ -118,9 +117,6 @@ public class IngestionServiceImplTest {
@Mock
private IFeatureFlag featureFlag;
@Mock
GcsVersionPathLimitationConfig gcsVersionPathLimitationConfig;
@Spy
CrcHashGenerator crcHashGenerator;
......@@ -1111,99 +1107,4 @@ public class IngestionServiceImplTest {
}
}
@Test
public void createUpdateRecords_shouldNotRemoveRecordsFromProcessing_ifGcsVersionPathsSizeIsLesserThanSoftLimit() {
when(featureFlag.isFeatureEnabled(OPA_FEATURE_NAME)).thenReturn(true);
when(gcsVersionPathLimitationConfig.getMaxLimit()).thenReturn(2000);
when(gcsVersionPathLimitationConfig.getFeatureName()).thenReturn("enforce_gcsVersionPathsLimit_enabled");
when(featureFlag.isFeatureEnabled(gcsVersionPathLimitationConfig.getFeatureName())).thenReturn(true);
when(this.authService.isValidAcl(any(), any())).thenReturn(true);
this.record1.setId(RECORD_ID1);
this.record1.setAcl(new Acl(VALID_ACL, VALID_ACL));
List<Record> processingRecords = new ArrayList<Record>();
processingRecords.add(this.record1);
RecordMetadata existingRecordMetadata1 = new RecordMetadata();
existingRecordMetadata1.setUser(NEW_USER);
existingRecordMetadata1.setKind(KIND_1);
existingRecordMetadata1.setStatus(RecordState.active);
existingRecordMetadata1.setAcl(new Acl(VALID_ACL, VALID_ACL));
List<String> versionsArray = IntStream.rangeClosed(1, 1999).mapToObj((item) -> {
return "path/" + item;
}).toList();
existingRecordMetadata1.setGcsVersionPaths(versionsArray);
Map<String, RecordMetadata> output = new HashMap<>();
output.put(RECORD_ID1, existingRecordMetadata1);
when(this.recordRepository.get(any(List.class), any())).thenReturn(output);
when(this.cloudStorage.read(existingRecordMetadata1, 1999L, false)).thenReturn(new Gson().toJson(this.record1));
TransferInfo transferInfo = this.sut.createUpdateRecords(false, processingRecords, USER, Optional.empty());
assertEquals(USER, transferInfo.getUser());
assertEquals(Integer.valueOf(1), transferInfo.getRecordCount());
assertNotNull(transferInfo.getVersion());
ArgumentCaptor<TransferBatch> transfer = ArgumentCaptor.forClass(TransferBatch.class);
verify(this.persistenceService, times(1)).persistRecordBatch(transfer.capture(), any());
verify(this.auditLogger).createOrUpdateRecordsSuccess(any());
TransferBatch input = transfer.getValue();
for (RecordProcessing rp : input.getRecords()) {
assertEquals(OperationType.update, rp.getOperationType());
}
}
@Test
public void createUpdateRecords_shouldRemoveRecordsFromProcessing_ifGcsSizeIsGreaterThanMaxLimit() {
when(featureFlag.isFeatureEnabled(OPA_FEATURE_NAME)).thenReturn(true);
when(gcsVersionPathLimitationConfig.getMaxLimit()).thenReturn(2000);
when(gcsVersionPathLimitationConfig.getFeatureName()).thenReturn("enforce_gcsVersionPathsLimit_enabled");
when(featureFlag.isFeatureEnabled(gcsVersionPathLimitationConfig.getFeatureName())).thenReturn(true);
when(this.authService.isValidAcl(any(), any())).thenReturn(true);
this.record1.setId(RECORD_ID1);
this.record1.setAcl(new Acl(VALID_ACL, VALID_ACL));
List<Record> processingRecords = new ArrayList<Record>();
processingRecords.add(this.record1);
RecordMetadata existingRecordMetadata1 = new RecordMetadata();
existingRecordMetadata1.setUser(NEW_USER);
existingRecordMetadata1.setKind(KIND_1);
existingRecordMetadata1.setStatus(RecordState.active);
existingRecordMetadata1.setAcl(new Acl(VALID_ACL, VALID_ACL));
List<String> versionsArray = IntStream.rangeClosed(0, 2000).mapToObj((item) -> {
return "path/" + item;
}).toList();
existingRecordMetadata1.setGcsVersionPaths(versionsArray);
Map<String, RecordMetadata> output = new HashMap<>();
output.put(RECORD_ID1, existingRecordMetadata1);
when(this.recordRepository.get(any(List.class), any())).thenReturn(output);
when(this.cloudStorage.read(existingRecordMetadata1, 2000L, false)).thenReturn(new Gson().toJson(this.record1));
TransferInfo transferInfo = this.sut.createUpdateRecords(false, processingRecords, USER, Optional.empty());
assertEquals(USER, transferInfo.getUser());
assertEquals(Integer.valueOf(1), transferInfo.getRecordCount());
assertNotNull(transferInfo.getVersion());
assertEquals(1, transferInfo.getSkippedRecords().size());
ArgumentCaptor<TransferBatch> transfer = ArgumentCaptor.forClass(TransferBatch.class);
verify(this.persistenceService, never()).persistRecordBatch(transfer.capture(), any());
verify(this.auditLogger, never()).createOrUpdateRecordsSuccess(any());
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment