diff --git a/NOTICE b/NOTICE index 1695a94bf8dfc7556915e0dd31b671a8342d0b17..b852f4b7a46f992818c4c45cbeef95fdc2298abf 100644 --- a/NOTICE +++ b/NOTICE @@ -55,7 +55,7 @@ The following software have components provided under the terms of this license: - Jackson-dataformat-YAML (from http://wiki.fasterxml.com/JacksonExtensionYAML, https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text) - Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names) - Jakarta Bean Validation API (from https://beanvalidation.org) -- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna, https://repo1.maven.org/maven2/net/java/dev/jna/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) @@ -191,10 +191,8 @@ EPL-1.0 The following software have components provided under the terms of this license: - JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) -- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) -- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) - Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) - Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) @@ -203,6 +201,8 @@ The following software have components provided under the terms of this license: - Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson) - Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core) - SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org) +- org.junit.jupiter:junit-jupiter-api (from http://junit.org/junit5/) +- org.junit.platform:junit-platform-commons (from http://junit.org/junit5/) ======================================================================== EPL-2.0 @@ -210,12 +210,12 @@ EPL-2.0 The following software have components provided under the terms of this license: - JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) -- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) -- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) - Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) +- org.junit.jupiter:junit-jupiter-api (from http://junit.org/junit5/) +- org.junit.platform:junit-platform-commons (from http://junit.org/junit5/) ======================================================================== GPL-2.0-only @@ -280,7 +280,7 @@ LGPL-2.1-only ======================================================================== The following software have components provided under the terms of this license: -- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna) +- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna, https://repo1.maven.org/maven2/net/java/dev/jna/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) - Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic) @@ -372,10 +372,10 @@ The following software have components provided under the terms of this license: - Checker Qual (from https://checkerframework.org) - JSON in Java (from https://github.com/douglascrockford/JSON-java) - JUnit Jupiter (Aggregator) (from https://junit.org/junit5/) -- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/) -- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/) - JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/) - Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf) - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) +- org.junit.jupiter:junit-jupiter-api (from http://junit.org/junit5/) +- org.junit.platform:junit-platform-commons (from http://junit.org/junit5/) diff --git a/pom.xml b/pom.xml index a100482691446edc61be7462a2a544823990e18d..535294a28c47285c64823be044f0f793a3cdf260 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 1.8 opendes UTF-8 - 0.13.0 + 0.14.0-rc3 2.11.2 4.1.51.Final 1.26 diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java index 5caef5953760255d6e11e3ff880e983241cd2cd3..1e8e68d9087c83c292ae7d359bb0a05063451cf9 100644 --- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java +++ b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java @@ -125,7 +125,7 @@ public class LegalComplianceChangeServiceAWSImpl implements ILegalComplianceChan ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase(compliantName)) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, RecordState.active); + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase(incompliantName)) { this.legalTagCache.delete(lt.getChangedTagName()); output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted); diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java index f3fadad4d2987b6486b45728e5485f647c6e7f9a..fe376ac0eae530615750ae11ae592a6e84f19091 100644 --- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java +++ b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java @@ -126,7 +126,7 @@ public class LegalComplianceChangeServiceAWSImplTest { // compliant pub sub info PubSubInfo compliantPubSubInfo = new PubSubInfo(); compliantPubSubInfo.setId(compliantRecordId); - compliantPubSubInfo.setOp(OperationType.create); + compliantPubSubInfo.setOp(OperationType.update); PubSubInfo[] compliantPubSubInfos = new PubSubInfo[1]; compliantPubSubInfos[0] = compliantPubSubInfo; diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/jobs/LegalComplianceChangeServiceAWSImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/jobs/LegalComplianceChangeServiceAWSImpl.java index a70bca6bbf09609d1505a25720e25cf299de9872..b3d850e7da65e06bb06e9e2bb02770f16a5b0ba1 100644 --- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/jobs/LegalComplianceChangeServiceAWSImpl.java +++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/jobs/LegalComplianceChangeServiceAWSImpl.java @@ -120,7 +120,7 @@ public class LegalComplianceChangeServiceAWSImpl implements ILegalComplianceChan ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase(compliantName)) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, RecordState.active); + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase(incompliantName)) { this.legalTagCache.delete(lt.getChangedTagName()); output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted); diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java index 1309bf8710a049cb8b3a3db449b13909ab8cc7b3..88282c012d0923b337e16cccd7805c7fd1af03cf 100644 --- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java +++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java @@ -124,7 +124,7 @@ public class LegalComplianceChangeServiceAWSImplTest { // compliant pub sub info PubSubInfo compliantPubSubInfo = new PubSubInfo(); compliantPubSubInfo.setId(compliantRecordId); - compliantPubSubInfo.setOp(OperationType.create); + compliantPubSubInfo.setOp(OperationType.update); PubSubInfo[] compliantPubSubInfos = new PubSubInfo[1]; compliantPubSubInfos[0] = compliantPubSubInfo; diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/LegalComplianceChangeServiceAzureImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/LegalComplianceChangeServiceAzureImpl.java index 42106898e12d4934a88d1e8a3b2cbd36f2b79386..621f001d88c276a53f6faadf36b00c9d2ec1ec9c 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/LegalComplianceChangeServiceAzureImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/LegalComplianceChangeServiceAzureImpl.java @@ -106,7 +106,7 @@ public class LegalComplianceChangeServiceAzureImpl implements ILegalComplianceCh ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, RecordState.active); + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) { this.legalTagCache.delete(lt.getChangedTagName()); output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted); diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/jobs/LegalComplianceChangeServiceGcpImpl.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/jobs/LegalComplianceChangeServiceGcpImpl.java index 680391c516fd9b59dde734c88cdb0146d82cf291..c41a74d917beb27ddb623690e932892f097ad90c 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/jobs/LegalComplianceChangeServiceGcpImpl.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/jobs/LegalComplianceChangeServiceGcpImpl.java @@ -118,7 +118,7 @@ public class LegalComplianceChangeServiceGcpImpl implements ILegalComplianceChan ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, RecordState.active); + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) { this.legalTagCache.delete(lt.getChangedTagName()); output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted); diff --git a/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/jobs/LegalComplianceChangeServiceImpl.java index 359bb72b5b11bed1b07401e54a337ce284c357e1..c5b24396e7697ae7ad471dc6972e030f588ffd43 100644 --- a/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/jobs/LegalComplianceChangeServiceImpl.java +++ b/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/jobs/LegalComplianceChangeServiceImpl.java @@ -119,7 +119,7 @@ public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeS ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase(compliantName)) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, RecordState.active); + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase(incompliantName)) { this.legalTagCache.delete(lt.getChangedTagName()); output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted); diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java index df5c6d90a8dcdd0e07e5f94ac5bd608a328e57cf..c4d0c4eae8492746c5a0a215000165f90ca06f05 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java @@ -126,7 +126,7 @@ public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeS ComplianceChangeInfo output = null; if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) { - output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active); } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) { this.legalTagCache.delete(lt.getChangedTagName()); diff --git a/storage-core/pom.xml b/storage-core/pom.xml index 4c700a3d7bdc17b08c2ad250a0bb92eacb13c081..bf491545581ae0571769bb82f8e6ea6f6068404f 100644 --- a/storage-core/pom.xml +++ b/storage-core/pom.xml @@ -34,7 +34,7 @@ 1.8 1.8 2.7.0 - 0.13.0 + 0.14.0-rc3 diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java index 42f4198d75ff6e4ef875b776bfc52fc271247601..e06b89eacf52d2e21fa433e9426b1a0c93383f96 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java @@ -192,6 +192,9 @@ public class IngestionServiceImpl implements IngestionService { } else { RecordMetadata existingRecordMetadata = existingRecords.get(record.getId()); RecordMetadata updatedRecordMetadata = new RecordMetadata(record); + if(!existingRecordMetadata.getKind().equalsIgnoreCase(updatedRecordMetadata.getKind())) { + updatedRecordMetadata.setPreviousVersionKind(existingRecordMetadata.getKind()); + } List versions = new ArrayList<>(); versions.addAll(existingRecordMetadata.getGcsVersionPaths()); diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/PersistenceServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/PersistenceServiceImpl.java index ed00654c3eb41adc3832d8ed7b716f329e6fc270..a585030490c99b04cf012ca38d4b26a0857f193d 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/PersistenceServiceImpl.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/PersistenceServiceImpl.java @@ -1,136 +1,144 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.storage.service; - -import org.apache.commons.lang3.NotImplementedException; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.entitlements.Acl; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.model.indexer.OperationType; -import org.opengroup.osdu.core.common.model.storage.*; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.storage.IPersistenceService; -import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; -import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; -import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -public class PersistenceServiceImpl implements IPersistenceService { - - @Autowired - private IRecordsMetadataRepository recordRepository; - - @Autowired - private ICloudStorage cloudStorage; - - @Autowired - private IMessageBus pubSubClient; - - @Autowired - private DpsHeaders headers; - - @Autowired - private JaxRsDpsLog logger; - - @Override - public void persistRecordBatch(TransferBatch transfer) { - - List recordsProcessing = transfer.getRecords(); - List recordsMetadata = new ArrayList<>(recordsProcessing.size()); - - PubSubInfo[] pubsubInfo = new PubSubInfo[recordsProcessing.size()]; - - for (int i = 0; i < recordsProcessing.size(); i++) { - RecordProcessing processing = recordsProcessing.get(i); - RecordMetadata recordMetadata = processing.getRecordMetadata(); - recordsMetadata.add(recordMetadata); - pubsubInfo[i] = new PubSubInfo(recordMetadata.getId(), recordMetadata.getKind(), OperationType.create); - } - - this.commitBatch(recordsProcessing, recordsMetadata); - this.pubSubClient.publishMessage(this.headers, pubsubInfo); - } - - private void commitBatch(List recordsProcessing, List recordsMetadata) { - - try { - this.commitCloudStorageTransaction(recordsProcessing); - this.commitDatastoreTransaction(recordsMetadata); - } catch (AppException e) { - try { - this.tryCleanupCloudStorage(recordsProcessing); - } catch (AppException innerException) { - e.addSuppressed(innerException); - } - - throw e; - } - } - - @Override - public List updateMetadata(List recordMetadata, List recordsId, Map recordsIdMap) { - Map originalAcls = new HashMap<>(); - List lockedRecords = new ArrayList<>(); - List validMetadata = new ArrayList<>(); - try { - originalAcls = this.cloudStorage.updateObjectMetadata(recordMetadata, recordsId, validMetadata, lockedRecords, recordsIdMap); - this.commitDatastoreTransaction(validMetadata); - } catch (NotImplementedException e) { - throw new AppException(HttpStatus.SC_NOT_IMPLEMENTED, "Not Implemented", "Interface not fully implemented yet"); - } catch (Exception e) { - this.logger.warning("Reverting meta data changes"); - try { - this.cloudStorage.revertObjectMetadata(recordMetadata, originalAcls); - } catch (NotImplementedException innerEx) { - throw new AppException(HttpStatus.SC_NOT_IMPLEMENTED, "Not Implemented", "Interface not fully implemented yet"); - } catch (Exception innerEx) { - e.addSuppressed(innerEx); - } - throw e; - } - PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()]; - for (int i = 0; i < recordMetadata.size(); i++) { - RecordMetadata metadata = recordMetadata.get(i); - pubsubInfo[i] = new PubSubInfo(metadata.getId(), metadata.getKind(), OperationType.create); - } - this.pubSubClient.publishMessage(this.headers, pubsubInfo); - return lockedRecords; - } - - private void tryCleanupCloudStorage(List recordsProcessing) { - recordsProcessing.forEach(r -> this.cloudStorage.deleteVersion(r.getRecordMetadata(), r.getRecordMetadata().getLatestVersion())); - } - - private void commitCloudStorageTransaction(List recordsProcessing) { - this.cloudStorage.write(recordsProcessing.toArray(new RecordProcessing[recordsProcessing.size()])); - } - - private void commitDatastoreTransaction(List recordsMetadata) { - try { - this.recordRepository.createOrUpdate(recordsMetadata); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error writing record.", - "The server could not process your request at the moment.", e); - } - } +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.storage.service; + +import com.google.common.base.Strings; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.entitlements.Acl; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.model.storage.*; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.storage.IPersistenceService; +import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; +import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class PersistenceServiceImpl implements IPersistenceService { + + @Autowired + private IRecordsMetadataRepository recordRepository; + + @Autowired + private ICloudStorage cloudStorage; + + @Autowired + private IMessageBus pubSubClient; + + @Autowired + private DpsHeaders headers; + + @Autowired + private JaxRsDpsLog logger; + + @Override + public void persistRecordBatch(TransferBatch transfer) { + + List recordsProcessing = transfer.getRecords(); + List recordsMetadata = new ArrayList<>(recordsProcessing.size()); + + PubSubInfo[] pubsubInfo = new PubSubInfo[recordsProcessing.size()]; + + for (int i = 0; i < recordsProcessing.size(); i++) { + RecordProcessing processing = recordsProcessing.get(i); + RecordMetadata recordMetadata = processing.getRecordMetadata(); + recordsMetadata.add(recordMetadata); + if(processing.getOperationType() == OperationType.create) { + pubsubInfo[i] = PubSubInfo.builder().id(recordMetadata.getId()).kind(recordMetadata.getKind()).op(OperationType.create).build(); + } else { + pubsubInfo[i] = PubSubInfo.builder().id(recordMetadata.getId()).kind(recordMetadata.getKind()).op(OperationType.update).build(); + if (!Strings.isNullOrEmpty(processing.getRecordMetadata().getPreviousVersionKind())) { + pubsubInfo[i].setPreviousVersionKind(processing.getRecordMetadata().getPreviousVersionKind()); + } + } + } + + this.commitBatch(recordsProcessing, recordsMetadata); + this.pubSubClient.publishMessage(this.headers, pubsubInfo); + } + + private void commitBatch(List recordsProcessing, List recordsMetadata) { + + try { + this.commitCloudStorageTransaction(recordsProcessing); + this.commitDatastoreTransaction(recordsMetadata); + } catch (AppException e) { + try { + this.tryCleanupCloudStorage(recordsProcessing); + } catch (AppException innerException) { + e.addSuppressed(innerException); + } + + throw e; + } + } + + @Override + public List updateMetadata(List recordMetadata, List recordsId, Map recordsIdMap) { + Map originalAcls = new HashMap<>(); + List lockedRecords = new ArrayList<>(); + List validMetadata = new ArrayList<>(); + try { + originalAcls = this.cloudStorage.updateObjectMetadata(recordMetadata, recordsId, validMetadata, lockedRecords, recordsIdMap); + this.commitDatastoreTransaction(validMetadata); + } catch (NotImplementedException e) { + throw new AppException(HttpStatus.SC_NOT_IMPLEMENTED, "Not Implemented", "Interface not fully implemented yet"); + } catch (Exception e) { + this.logger.warning("Reverting meta data changes"); + try { + this.cloudStorage.revertObjectMetadata(recordMetadata, originalAcls); + } catch (NotImplementedException innerEx) { + throw new AppException(HttpStatus.SC_NOT_IMPLEMENTED, "Not Implemented", "Interface not fully implemented yet"); + } catch (Exception innerEx) { + e.addSuppressed(innerEx); + } + throw e; + } + PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()]; + for (int i = 0; i < recordMetadata.size(); i++) { + RecordMetadata metadata = recordMetadata.get(i); + pubsubInfo[i] = new PubSubInfo(metadata.getId(), metadata.getKind(), OperationType.update); + } + this.pubSubClient.publishMessage(this.headers, pubsubInfo); + return lockedRecords; + } + + private void tryCleanupCloudStorage(List recordsProcessing) { + recordsProcessing.forEach(r -> this.cloudStorage.deleteVersion(r.getRecordMetadata(), r.getRecordMetadata().getLatestVersion())); + } + + private void commitCloudStorageTransaction(List recordsProcessing) { + this.cloudStorage.write(recordsProcessing.toArray(new RecordProcessing[recordsProcessing.size()])); + } + + private void commitDatastoreTransaction(List recordsMetadata) { + try { + this.recordRepository.createOrUpdate(recordsMetadata); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error writing record.", + "The server could not process your request at the moment.", e); + } + } } \ No newline at end of file diff --git a/storage-core/src/test/java/org/opengroup/osdu/storage/service/IngestionServiceImplTest.java b/storage-core/src/test/java/org/opengroup/osdu/storage/service/IngestionServiceImplTest.java index 1a49610e480bc292feea058dbb1c391145497f3e..1fa1acf64d9982b9e52490d11e16fda61c8be91a 100644 --- a/storage-core/src/test/java/org/opengroup/osdu/storage/service/IngestionServiceImplTest.java +++ b/storage-core/src/test/java/org/opengroup/osdu/storage/service/IngestionServiceImplTest.java @@ -385,6 +385,50 @@ public class IngestionServiceImplTest { } } + @Test + @SuppressWarnings("unchecked") + public void should_includePriorKind_when_kindUpdated() { + when(this.authService.isValidAcl(any(), any())).thenReturn(true); + + this.record1.setId(RECORD_ID1); + this.record1.setKind(KIND_2); + this.acl.setViewers(VALID_ACL); + this.acl.setOwners(VALID_ACL); + + RecordMetadata existingRecordMetadata1 = new RecordMetadata(); + existingRecordMetadata1.setUser(NEW_USER); + existingRecordMetadata1.setKind(KIND_1); + existingRecordMetadata1.setStatus(RecordState.active); + existingRecordMetadata1.setAcl(this.acl); + existingRecordMetadata1.setGcsVersionPaths(Lists.newArrayList("path/1", "path/2", "path/3")); + + Map output = new HashMap<>(); + output.put(RECORD_ID1, existingRecordMetadata1); + + when(this.cloudStorage.hasAccess(existingRecordMetadata1)).thenReturn(true); + + when(this.recordRepository.get(any(List.class))).thenReturn(output); + + when(this.authService.hasOwnerAccess(any(), any())).thenReturn(true); + + TransferInfo transferInfo = this.sut.createUpdateRecords(false, Collections.singletonList(this.record1), USER); + assertEquals(USER, transferInfo.getUser()); + assertEquals(new Integer(1), transferInfo.getRecordCount()); + assertNotNull(transferInfo.getVersion()); + + ArgumentCaptor transfer = ArgumentCaptor.forClass(TransferBatch.class); + + verify(this.persistenceService, times(1)).persistRecordBatch(transfer.capture()); + verify(this.auditLogger).createOrUpdateRecordsSuccess(any()); + + TransferBatch input = transfer.getValue(); + + for (RecordProcessing rp : input.getRecords()) { + assertEquals(OperationType.update, rp.getOperationType()); + assertEquals(KIND_1, rp.getRecordMetadata().getPreviousVersionKind()); + } + } + @Test @SuppressWarnings("unchecked") public void should_disregardUpdateRecord_when_skipDupesAndSameRecordContent() { diff --git a/storage-core/src/test/java/org/opengroup/osdu/storage/service/PersistenceServiceImplTest.java b/storage-core/src/test/java/org/opengroup/osdu/storage/service/PersistenceServiceImplTest.java index e0149773dbd711737413559de9e1077089d5c641..f0759ade02e6fce17eca65518960a30dacf6ddc2 100644 --- a/storage-core/src/test/java/org/opengroup/osdu/storage/service/PersistenceServiceImplTest.java +++ b/storage-core/src/test/java/org/opengroup/osdu/storage/service/PersistenceServiceImplTest.java @@ -323,9 +323,11 @@ public class PersistenceServiceImplTest { assertEquals(successfullRecords, pubsubList.size()); - for (PubSubInfo pubSubInfo : pubsubList) { + for (int i = 0; i < pubsubList.size(); i++) { + PubSubInfo pubSubInfo = pubsubList.get(i); assertEquals("anyKind", pubSubInfo.getKind()); - assertEquals(OperationType.create, pubSubInfo.getOp()); + assertEquals(i % 2 == 0 ? OperationType.create : OperationType.update, pubSubInfo.getOp()); + assertNull(pubSubInfo.getPreviousVersionKind()); assertTrue(pubSubInfo.getId().startsWith("ID")); } }