From 3223a3e114e8b8a62c19854a23a24650d586f355 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Wed, 7 Oct 2020 12:16:11 +0400 Subject: [PATCH 01/44] add publish schema purge --- .../org/opengroup/osdu/storage/service/SchemaServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/SchemaServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/SchemaServiceImpl.java index ea0ecb62..2dac6209 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/SchemaServiceImpl.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/SchemaServiceImpl.java @@ -118,6 +118,8 @@ public class SchemaServiceImpl implements SchemaService { this.auditLogger.deleteSchemaSuccess(singletonList(schema.getKind())); this.cache.delete(this.getSchemaCacheKey(kind)); + this.pubSubClient.publishMessage(this.headers, + new PubSubInfo(null, schema.getKind(), OperationType.purge_schema)); } @Override -- GitLab From e1935f1fd8d8c6838625679cd6f2ebec6e60c203 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Fri, 16 Oct 2020 11:02:40 +0400 Subject: [PATCH 02/44] GONRG-941 Added an audit event 'Query all kinds' for all providers. --- .../storage/provider/aws/service/BatchServiceAWSImpl.java | 2 +- .../storage/provider/azure/service/BatchServiceAzureImpl.java | 2 +- .../storage/provider/byoc/service/BatchServiceByocImpl.java | 2 +- .../storage/provider/gcp/service/BatchServiceGcpImpl.java | 2 +- .../storage/provider/ibm/service/BatchServiceIBMImpl.java | 2 +- .../opengroup/osdu/storage/logging/StorageAuditEvents.java | 4 ++-- .../opengroup/osdu/storage/logging/StorageAuditLogger.java | 4 ++-- .../org/opengroup/osdu/storage/logging/AuditLoggerTest.java | 3 ++- 8 files changed, 11 insertions(+), 10 deletions(-) diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/service/BatchServiceAWSImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/service/BatchServiceAWSImpl.java index 1fd3fd44..fd7ac051 100644 --- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/service/BatchServiceAWSImpl.java +++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/service/BatchServiceAWSImpl.java @@ -44,7 +44,7 @@ public class BatchServiceAWSImpl extends BatchServiceImpl { { try { DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); - this.auditLogger.readAllKindsSuccess(); + this.auditLogger.readAllKindsSuccess(result.getResults()); return result; } catch (InvalidCursorException e) { throw new AppException(HttpStatus.SC_BAD_REQUEST, "Cursor invalid", "The requested cursor does not exist or is invalid", e); diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/BatchServiceAzureImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/BatchServiceAzureImpl.java index a6645afa..f843dcf3 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/BatchServiceAzureImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/service/BatchServiceAzureImpl.java @@ -39,7 +39,7 @@ public class BatchServiceAzureImpl extends BatchServiceImpl { public DatastoreQueryResult getAllKinds(String cursor, Integer limit) { DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); - this.auditLogger.readAllKindsSuccess(); + this.auditLogger.readAllKindsSuccess(result.getResults()); return result; } diff --git a/provider/storage-byoc/src/main/java/org/opengroup/osdu/storage/provider/byoc/service/BatchServiceByocImpl.java b/provider/storage-byoc/src/main/java/org/opengroup/osdu/storage/provider/byoc/service/BatchServiceByocImpl.java index 869f7f3e..57c9f5f7 100644 --- a/provider/storage-byoc/src/main/java/org/opengroup/osdu/storage/provider/byoc/service/BatchServiceByocImpl.java +++ b/provider/storage-byoc/src/main/java/org/opengroup/osdu/storage/provider/byoc/service/BatchServiceByocImpl.java @@ -40,7 +40,7 @@ public class BatchServiceByocImpl extends BatchServiceImpl { { try { DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); - this.auditLogger.readAllKindsSuccess(); + this.auditLogger.readAllKindsSuccess(result.getResults()); return result; } catch (Exception e) { throw this.getInternalErrorException(); diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/service/BatchServiceGcpImpl.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/service/BatchServiceGcpImpl.java index 43342935..0d72bcdc 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/service/BatchServiceGcpImpl.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/service/BatchServiceGcpImpl.java @@ -42,7 +42,7 @@ public class BatchServiceGcpImpl extends BatchServiceImpl { try { DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); - this.auditLogger.readAllKindsSuccess(); + this.auditLogger.readAllKindsSuccess(result.getResults()); return result; } catch (DatastoreException e) { // Is invalid cursor (code = 3) exception? diff --git a/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/service/BatchServiceIBMImpl.java b/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/service/BatchServiceIBMImpl.java index 8b9426c7..c1ea234f 100644 --- a/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/service/BatchServiceIBMImpl.java +++ b/provider/storage-ibm/src/main/java/org/opengroup/osdu/storage/provider/ibm/service/BatchServiceIBMImpl.java @@ -41,7 +41,7 @@ public class BatchServiceIBMImpl extends BatchServiceImpl { public DatastoreQueryResult getAllKinds(String cursor, Integer limit) { DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); - this.auditLogger.readAllKindsSuccess(); + this.auditLogger.readAllKindsSuccess(result.getResults()); return result; } diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditEvents.java b/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditEvents.java index 52508a81..e34e5562 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditEvents.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditEvents.java @@ -222,13 +222,13 @@ public class StorageAuditEvents { } - public AuditPayload getAllKindsEventSuccess() { + public AuditPayload getAllKindsEventSuccess(List resource) { return AuditPayload.builder() .action(AuditAction.READ) .status(AuditStatus.SUCCESS) .actionId(READ_GET_ALL_KINDS_ACTION_ID) .message(READ_GET_ALL_KINDS_MESSAGE) - .resources(singletonList("All Kinds")) + .resources(resource) .user(user) .build(); } diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditLogger.java b/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditLogger.java index a37948df..b64fbdc0 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditLogger.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/logging/StorageAuditLogger.java @@ -98,8 +98,8 @@ public class StorageAuditLogger { this.writeLog(this.getAuditEvents().getReadAllRecordsOfGivenKindSuccess(resource)); } - public void readAllKindsSuccess() { - this.writeLog(this.getAuditEvents().getAllKindsEventSuccess()); + public void readAllKindsSuccess(List resource) { + this.writeLog(this.getAuditEvents().getAllKindsEventSuccess(resource)); } public void createSchemaSuccess(List resource) { diff --git a/storage-core/src/test/java/org/opengroup/osdu/storage/logging/AuditLoggerTest.java b/storage-core/src/test/java/org/opengroup/osdu/storage/logging/AuditLoggerTest.java index bfcf2819..62a6a979 100644 --- a/storage-core/src/test/java/org/opengroup/osdu/storage/logging/AuditLoggerTest.java +++ b/storage-core/src/test/java/org/opengroup/osdu/storage/logging/AuditLoggerTest.java @@ -125,7 +125,8 @@ public class AuditLoggerTest { @Test public void should_writeReadAllKindsEvent() { - this.sut.readAllKindsSuccess(); + List resource = Collections.singletonList("1"); + this.sut.readAllKindsSuccess(resource); verify(this.log).audit(any()); } -- GitLab From 393f5d2a5979cf696a9fb69bab73a60b5aee85e0 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Tue, 20 Oct 2020 18:20:47 +0400 Subject: [PATCH 03/44] Storage Service for Anthos PoC --- pom.xml | 1 + provider/storage-reference/pom.xml | 130 ++++++++++++++ .../provider/reference/CloudStorageImpl.java | 164 ++++++++++++++++++ .../app/StorageReferenceApplication.java | 16 ++ .../provider/reference/cache/GroupCache.java | 13 ++ .../reference/cache/LegalTagCache.java | 47 +++++ .../provider/reference/cache/SchemaCache.java | 13 ++ .../reference/di/RabbitMQFactoryImpl.java | 65 +++++++ .../reference/di/TenantFactoryImpl.java | 85 +++++++++ .../LegalComplianceChangeServiceImpl.java | 120 +++++++++++++ .../reference/messagebus/IMessageFactory.java | 13 ++ .../model/RecordMetadataDocument.java | 31 ++++ .../reference/model/SchemaDocument.java | 65 +++++++ .../reference/model/TenantInfoDocument.java | 17 ++ .../persistence/MongoDdmsClient.java | 23 +++ .../repository/QueryRepositoryImpl.java | 80 +++++++++ .../RecordsMetadataRepositoryImpl.java | 153 ++++++++++++++++ .../repository/SchemaRepositoryImpl.java | 73 ++++++++ .../security/BasicAuthSecurityConfig.java | 18 ++ .../reference/security/WhoamiController.java | 25 +++ .../service/BatchServiceReferenceImpl.java | 37 ++++ .../reference/service/MessageBusImpl.java | 39 +++++ .../reference/util/MongoClientHandler.java | 68 ++++++++ .../util/ServiceAccountJwtClientImpl.java | 14 ++ .../src/main/resources/application.properties | 17 ++ .../context/ProviderSpringContextTest.java | 16 ++ .../src/test/resources/application.properties | 17 ++ 27 files changed, 1360 insertions(+) create mode 100644 provider/storage-reference/pom.xml create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java create mode 100644 provider/storage-reference/src/main/resources/application.properties create mode 100644 provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java create mode 100644 provider/storage-reference/src/test/resources/application.properties diff --git a/pom.xml b/pom.xml index 3a9c6ad6..637dd393 100644 --- a/pom.xml +++ b/pom.xml @@ -119,6 +119,7 @@ provider/storage-azure provider/storage-aws provider/storage-ibm + provider/storage-reference diff --git a/provider/storage-reference/pom.xml b/provider/storage-reference/pom.xml new file mode 100644 index 00000000..ebade6bc --- /dev/null +++ b/provider/storage-reference/pom.xml @@ -0,0 +1,130 @@ + + + + + + os-storage + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../.. + + 4.0.0 + + storage-reference + jar + + + + org.opengroup.osdu + os-core-common + + + org.opengroup.osdu + storage-core + 0.0.5-SNAPSHOT + + + + junit + junit + 4.12 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + org.powermock + powermock-module-junit4 + 2.0.2 + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.security + spring-security-test + test + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + org.springframework.security + spring-security-oauth2-jose + + + org.springframework.security + spring-security-oauth2-resource-server + + + com.rabbitmq + amqp-client + 5.7.3 + + + + + + + ${gitlab-server} + https://community.opengroup.org/api/v4/groups/17/-/packages/maven + + + + + + ${gitlab-server} + https://community.opengroup.org/api/v4/projects/44/packages/maven + + + ${gitlab-server} + https://community.opengroup.org/api/v4/projects/44/packages/maven + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + spring-boot + + org.opengroup.osdu.storage.provider.reference.app.StorageReferenceApplication + + + + + + + + + \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java new file mode 100644 index 00000000..b5db09b2 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java @@ -0,0 +1,164 @@ +package org.opengroup.osdu.storage.provider.reference; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import org.opengroup.osdu.core.common.model.storage.*; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.util.Crc32c; +import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; +import org.apache.http.HttpStatus; +import org.springframework.stereotype.Repository; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static org.apache.commons.codec.binary.Base64.encodeBase64; + +@Repository +public class CloudStorageImpl implements ICloudStorage { + private static List memList = new ArrayList<>(); + + @Override + public void write(RecordProcessing... recordsProcessing){ + for (RecordProcessing rp: recordsProcessing) { + memList.add(rp); + } + } + + @Override + public Map getHash(Collection records) + { + Gson gson = new Gson(); + Map hashes = new HashMap<>(); + for (RecordMetadata rm : records) + { + String jsonData = this.read(rm, rm.getLatestVersion(), false); + RecordData data = gson.fromJson(jsonData, RecordData.class); + + String hash = getHash(data); + hashes.put(rm.getId(), hash); + } + return hashes; + } + + @Override + public boolean isDuplicateRecord(TransferInfo transfer, Map hashMap, Map.Entry kv) { + RecordMetadata updatedRecordMetadata = kv.getKey(); + RecordData recordData = kv.getValue(); + String recordHash = hashMap.get(updatedRecordMetadata.getId()); + + String newHash = getHash(recordData); + + if (newHash.equals(recordHash)) { + transfer.getSkippedRecords().add(updatedRecordMetadata.getId()); + return true; + }else{ + return false; + } + } + + private String getHash(RecordData data) { + Gson gson = new Gson(); + Crc32c checksumGenerator = new Crc32c(); + + String newRecordStr = gson.toJson(data); + byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8); + checksumGenerator.update(bytes, 0, bytes.length); + bytes = checksumGenerator.getValueAsBytes(); + String newHash = new String(encodeBase64(bytes)); + return newHash; + } + + @Override + public void delete(RecordMetadata record) + { + Iterator it = memList.iterator(); + while(it.hasNext()) + { + RecordProcessing rp = it.next(); + if (rp.getRecordMetadata().getId().equals(record.getId())) + { + it.remove(); + } + } + } + + @Override + public void deleteVersion(RecordMetadata record, Long version) { + //TODO: Check if this is correct + + Iterator it = memList.iterator(); + while(it.hasNext()) + { + RecordProcessing rp = it.next(); + if (rp.getRecordMetadata().getId().equals(record.getId())) { + for (String path : rp.getRecordMetadata().getGcsVersionPaths()) { + if (path.contains(version.toString())) { + it.remove(); + } + } + } + } + } + + @Override + public boolean hasAccess(RecordMetadata... records) + { + for (RecordMetadata record : records) + { + if (!record.getStatus().equals(RecordState.active)) + return false; + } + return true; + } + + @Override + public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) + { + for (RecordProcessing rp : memList) + { + RecordMetadata rmd = rp.getRecordMetadata(); + if (rmd.getId().equals(record.getId())) + { + for (String path: rmd.getGcsVersionPaths()) + { + if (path.contains(version.toString())) { + try { + String result = new ObjectMapper().writeValueAsString(rp.getRecordData()); + return result; + } catch (JsonProcessingException je) + { + throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", je.getMessage()); + } + } + } + } + } + String msg = String.format("Record with id '%s' does not exist", record.getId()); + throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg); + } + + @Override + public Map read(Map objects) + { + Map map = new HashMap<>(); + for (RecordProcessing rp : memList) + { + RecordMetadata rmd = rp.getRecordMetadata(); + String id = rmd.getId(); + if (objects.containsKey(id)) + { + try { + String result = new ObjectMapper().writeValueAsString(rp.getRecordData()); + map.put(id, result); + } catch (JsonProcessingException je) + { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Record to json failed", je.getMessage()); + } + } + } + return map; + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java new file mode 100644 index 00000000..595e3ebd --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java @@ -0,0 +1,16 @@ +package org.opengroup.osdu.storage.provider.reference.app; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; + +@SpringBootApplication +@ComponentScan({"org.opengroup.osdu"}) +public class StorageReferenceApplication { + + public static void main(String[] args) { + + SpringApplication.run(StorageReferenceApplication.class, args); + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java new file mode 100644 index 00000000..c94499f5 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java @@ -0,0 +1,13 @@ +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.model.entitlements.Groups; +import org.springframework.stereotype.Component; + +@Component +public class GroupCache extends VmCache { + + public GroupCache() { + super(5 * 60, 1000); + } +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java new file mode 100644 index 00000000..685c21b9 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java @@ -0,0 +1,47 @@ +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.cache.MultiTenantCache; +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component("LegalTagCache") +public class LegalTagCache implements ICache { + + @Autowired + private TenantInfo tenant; + + private final MultiTenantCache caches; + + public LegalTagCache() { + this.caches = new MultiTenantCache<>( + new VmCache(60 * 60, 1000)); + } + + @Override + public void put(String key, String val) { + this.partitionCache().put(key, val); + } + + @Override + public String get(String key) { + return this.partitionCache().get(key); + } + + @Override + public void delete(String key) { + this.partitionCache().delete(key); + } + + @Override + public void clearAll() { + this.partitionCache().clearAll(); + } + + private ICache partitionCache() { + return this.caches.get(String.format("%s:legalTag", this.tenant)); + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java new file mode 100644 index 00000000..deda3ff2 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java @@ -0,0 +1,13 @@ +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.springframework.stereotype.Component; + +@Component +public class SchemaCache extends VmCache { + + public SchemaCache() { + super(5 * 60, 1000); + } +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java new file mode 100644 index 00000000..69bf9d38 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java @@ -0,0 +1,65 @@ +package org.opengroup.osdu.storage.provider.reference.di; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; +import javax.annotation.PostConstruct; +import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Component +public class RabbitMQFactoryImpl implements IMessageFactory { + + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + + @Value("${mb.rabbitmq.uri}") + private String uri; + + private Channel channel; + + @PostConstruct + private void init() { + ConnectionFactory factory = new ConnectionFactory(); + try { + factory.setUri(uri); + factory.setAutomaticRecoveryEnabled(true); + Connection conn = factory.newConnection(); + this.channel = conn.createChannel(); + LOG.debug("RabbitMQ Channel was created."); + for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { + channel.queueDeclare("os-storage-" + queue, false, false, false, null); + LOG.debug("Queue [os-storage-" + queue + "] was declared."); + } + } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { + LOG.error(e.getMessage(), e); + } + } + + @Override + public void sendMessage(String msg) { + sendMessage(DEFAULT_QUEUE_NAME, msg); + } + + @Override + public void sendMessage(String queueName, String msg) { + String queueNameWithPrefix = "os-storage-" + queueName; + try { + channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); + LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); + } catch (IOException e) { + LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]"); + LOG.error(e.getMessage(), e); + } + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java new file mode 100644 index 00000000..0a28547e --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java @@ -0,0 +1,85 @@ +package org.opengroup.osdu.storage.provider.reference.di; + +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.bson.Document; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; +import org.opengroup.osdu.storage.provider.reference.model.TenantInfoDocument; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TenantFactoryImpl implements ITenantFactory { + + private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); + public static final String TENANT_INFO = "TenantInfo"; + + @Autowired + private MongoDdmsClient mongoClient; + + private Map tenants; + + public boolean exists(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.containsKey(tenantName); + } + + public TenantInfo getTenantInfo(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.get(tenantName); + } + + public Collection listTenantInfo() { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.values(); + } + + public ICache createCache(String tenantName, String host, int port, + int expireTimeSeconds, Class classOfV) { + return null; + } + + public void flushCache() { + } + + private void initTenants() { + this.tenants = new HashMap<>(); + MongoCollection mongoCollection = mongoClient + .getMongoCollection(SCHEMA_DATABASE, TENANT_INFO); + FindIterable results = mongoCollection.find(); + if (Objects.isNull(results) && Objects.isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + } + for (Document document : results) { + TenantInfoDocument tenantInfoDocument = new Gson() + .fromJson(document.toJson(), TenantInfoDocument.class); + TenantInfo tenantInfo = convertToTenantInfo(tenantInfoDocument); + this.tenants.put(tenantInfo.getName(), tenantInfo); + } + } + + private TenantInfo convertToTenantInfo(TenantInfoDocument tenantInfoDocument) { + TenantInfo tenantInfo = new TenantInfo(); + tenantInfo.setName(tenantInfoDocument.getId()); + return tenantInfo; + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java new file mode 100644 index 00000000..61ec94b1 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java @@ -0,0 +1,120 @@ +package org.opengroup.osdu.storage.provider.reference.jobs; + +import static java.util.Collections.singletonList; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.model.legal.LegalCompliance; +import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceChangeInfo; +import org.opengroup.osdu.core.common.model.legal.jobs.ILegalComplianceChangeService; +import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChanged; +import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection; +import org.opengroup.osdu.core.common.model.storage.PubSubInfo; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.opengroup.osdu.storage.logging.StorageAuditLogger; +import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.reference.cache.LegalTagCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeService { + + private static final Logger LOG = LoggerFactory.getLogger(LegalComplianceChangeServiceImpl.class); + + @Autowired + private IRecordsMetadataRepository recordsRepo; + + @Autowired + private IMessageBus pubSubclient; + + @Autowired + private StorageAuditLogger auditLogger; + + @Autowired + private LegalTagCache legalTagCache; + + @Override + public Map updateComplianceOnRecords( + LegalTagChangedCollection legalTagsChanged, + DpsHeaders headers) { + Map output = new HashMap<>(); + + for (LegalTagChanged lt : legalTagsChanged.getStatusChangedTags()) { + + ComplianceChangeInfo complianceChangeInfo = this.getComplianceChangeInfo(lt); + if (complianceChangeInfo == null) { + continue; + } + + String cursor = null; + do { + //TODO replace with the new method queryByLegal + AbstractMap.SimpleEntry> results = this.recordsRepo + .queryByLegalTagName(lt.getChangedTagName(), 500, cursor); + cursor = results.getKey(); + + if (results.getValue() != null && !results.getValue().isEmpty()) { + List recordsMetadata = results.getValue(); + PubSubInfo[] pubsubInfos = this + .updateComplianceStatus(complianceChangeInfo, recordsMetadata, output); + this.recordsRepo.createOrUpdate(recordsMetadata); + StringBuilder recordsId = new StringBuilder(); + for (RecordMetadata recordMetadata : recordsMetadata) { + recordsId.append(", ").append(recordMetadata.getId()); + } + this.auditLogger.updateRecordsComplianceStateSuccess( + singletonList("[" + recordsId.toString().substring(2) + "]")); + + this.pubSubclient.publishMessage(headers, pubsubInfos); + } + } while (cursor != null); + } + + return output; + } + + private PubSubInfo[] updateComplianceStatus(ComplianceChangeInfo complianceChangeInfo, + List recordMetadata, Map output) { + + PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()]; + + int i = 0; + for (RecordMetadata rm : recordMetadata) { + rm.getLegal().setStatus(complianceChangeInfo.getNewState()); + rm.setStatus(complianceChangeInfo.getNewRecordState()); + pubsubInfo[i] = new PubSubInfo(rm.getId(), rm.getKind(), + complianceChangeInfo.getPubSubEvent()); + output.put(rm.getId(), complianceChangeInfo.getNewState()); + i++; + } + + return pubsubInfo; + } + + private ComplianceChangeInfo getComplianceChangeInfo(LegalTagChanged lt) { + ComplianceChangeInfo output = null; + + if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) { + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, + RecordState.active); + } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) { + this.legalTagCache.delete(lt.getChangedTagName()); + output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, + RecordState.deleted); + } else { + LOG.warn(String.format("Unknown LegalTag compliance status received %s %s", + lt.getChangedTagStatus(), lt.getChangedTagName())); + } + + return output; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java new file mode 100644 index 00000000..f55db10a --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java @@ -0,0 +1,13 @@ +package org.opengroup.osdu.storage.provider.reference.messagebus; + +public interface IMessageFactory { + + String DEFAULT_QUEUE_NAME = "records"; + String LEGAL_QUEUE_NAME = "legal"; + String INDEXER_QUEUE_NAME = "indexer"; + + void sendMessage(String msg); + + void sendMessage(String queueName, String msg); + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java new file mode 100644 index 00000000..a38f98f1 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java @@ -0,0 +1,31 @@ +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.ArrayList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.opengroup.osdu.core.common.model.entitlements.Acl; +import org.opengroup.osdu.core.common.model.legal.Legal; +import org.opengroup.osdu.core.common.model.storage.RecordAncestry; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.springframework.data.mongodb.core.mapping.Document; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Document(collection = "StorageRecord") +public class RecordMetadataDocument { + + private String id; + private String kind; + private Acl acl; + private Legal legal; + private RecordAncestry ancestry; + private List gcsVersionPaths = new ArrayList(); + private RecordState status; + private String user; + private Long createTime; + private String modifyUser; + private Long modifyTime; +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java new file mode 100644 index 00000000..8a010588 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java @@ -0,0 +1,65 @@ +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.Map; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.opengroup.osdu.core.common.model.storage.SchemaItem; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "StorageSchema") +public class SchemaDocument { + + private String kind; + private String rev; + private String user; + private SchemaItem[] schema; + private Map extension; + + + public SchemaDocument(Schema schema, String user) { + this.setKind(schema.getKind()); + this.setExtension(schema.getExt()); + this.setSchema(schema.getSchema()); + this.setUser(user); + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + public Map getExtension() { + return extension; + } + + public void setExtension(Map extension) { + this.extension = extension; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public SchemaItem[] getSchema() { + return schema; + } + + public void setSchema(SchemaItem[] schemaItems) { + this.schema = schemaItems; + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java new file mode 100644 index 00000000..f96066c8 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java @@ -0,0 +1,17 @@ +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.mongodb.core.mapping.Document; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Document(collection = "TenantInfo") +public class TenantInfoDocument { + + private String id; + private List groups; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java new file mode 100644 index 00000000..30836e90 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java @@ -0,0 +1,23 @@ +package org.opengroup.osdu.storage.provider.reference.persistence; + +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.storage.provider.reference.util.MongoClientHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MongoDdmsClient { + + @Autowired + private MongoClientHandler mongoClientHandler; + + @Autowired + private TenantInfo tenantInfo; + + public MongoCollection getMongoCollection(String dbName, String collectionName) { + return mongoClientHandler.getMongoClient().getDatabase(dbName) + .getCollection(collectionName); + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java new file mode 100644 index 00000000..9f03d474 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java @@ -0,0 +1,80 @@ +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.RECORD_STORAGE; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_STORAGE; + +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.ArrayList; +import java.util.List; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; +import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class QueryRepositoryImpl implements IQueryRepository { + + @Autowired + private IRecordsMetadataRepository recordsMetadataRepository; + + @Autowired + private ISchemaRepository schemaRepository; + + @Autowired + private MongoDdmsClient mongoDdmsClient; + + @Override + public DatastoreQueryResult getAllKinds(Integer limit, String cursor) { + int numRecords = PAGE_SIZE; + if (limit != null) { + numRecords = limit > 0 ? limit : PAGE_SIZE; + } + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + FindIterable results = mongoCollection.find() + .limit(numRecords); + List kinds = new ArrayList<>(); + for (Document document : results) { + kinds.add(document.get("kind").toString()); + } + return new DatastoreQueryResult(cursor, kinds); + } + + @Override + public DatastoreQueryResult getAllRecordIdsFromKind( + String kind, Integer limit, String cursor) { + boolean paginated = false; + + int numRecords = PAGE_SIZE; + if (limit != null) { + numRecords = limit > 0 ? limit : PAGE_SIZE; + paginated = true; + } + + if (cursor != null && !cursor.isEmpty()) { + paginated = true; + } + + DatastoreQueryResult dqr = new DatastoreQueryResult(); + List ids = new ArrayList(); + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, RECORD_STORAGE); + FindIterable results = mongoCollection.find(eq("kind", kind)) + .limit(numRecords); + for (Document document : results) { + ids.add(document.get("id").toString()); + } + dqr.setResults(ids); + dqr.setCursor(cursor); + return dqr; + } + +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java new file mode 100644 index 00000000..88eb82d0 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java @@ -0,0 +1,153 @@ +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.util.JSON.serialize; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.UpdateOptions; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.legal.LegalCompliance; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.reference.model.RecordMetadataDocument; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository { + + public static final String STORAGE_RECORD = "StorageRecord"; + + @Autowired + private MongoDdmsClient mongoDdmsClient; + + @Override + public List createOrUpdate(List recordsMetadata) { + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + if (recordsMetadata != null) { + for (RecordMetadata recordMetadata : recordsMetadata) { + RecordMetadataDocument recordMetadataDocument = convertToRecordMetadataDocument( + recordMetadata); + mongoCollection.replaceOne(eq("id", recordMetadataDocument.getId()), + Document.parse(new Gson().toJson(recordMetadataDocument)), + (new UpdateOptions()).upsert(true)); + } + } + return recordsMetadata; + } + + @Override + public void delete(String id) { + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + mongoCollection.deleteOne(eq("id", id)); + } + + @Override + public RecordMetadata get(String id) { + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + Document doc = mongoCollection.find(eq("id", id)).first(); + if (Objects.isNull(doc)) { + return null; + } + RecordMetadataDocument recordMetadataDocument = new Gson() + .fromJson(serialize(doc), RecordMetadataDocument.class); + return convertToRecordMetadata(recordMetadataDocument); + } + + @Override + public Map get(List ids) { + Map output = new HashMap<>(); + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + for (String id : ids) { + Document document = mongoCollection.find(eq("id", id)).first(); + RecordMetadataDocument recordMetadataDocument = null; + if (Objects.nonNull(document)) { + recordMetadataDocument = new Gson() + .fromJson(serialize(document), RecordMetadataDocument.class); + } + RecordMetadata rmd = convertToRecordMetadata(recordMetadataDocument); + if (Objects.isNull(rmd)) { + continue; + } + output.put(id, rmd); + } + return output; + } + + @Override + public AbstractMap.SimpleEntry> queryByLegalTagName( + String legalTagName, int limit, String cursor) { + MongoCollection mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + List outputRecords = new ArrayList<>(); + FindIterable results = mongoCollection.find().skip(limit * (limit - 1)).limit(limit); + for (Document document : results) { + RecordMetadataDocument recordMetadataDocument = new Gson() + .fromJson(serialize(document), RecordMetadataDocument.class); + if (Objects.nonNull(recordMetadataDocument)) { + if (recordMetadataDocument.getLegal().getLegaltags().contains(legalTagName)) { + RecordMetadata recordMetadata = convertToRecordMetadata(recordMetadataDocument); + outputRecords.add(recordMetadata); + } + } + } + return new AbstractMap.SimpleEntry<>(cursor, outputRecords); + } + + @Override + public AbstractMap.SimpleEntry> queryByLegal(String legalTagName, LegalCompliance status, int limit) { + return null; + } + + private RecordMetadataDocument convertToRecordMetadataDocument(RecordMetadata recordMetadata) { + RecordMetadataDocument recordMetadataDocument = new RecordMetadataDocument(); + recordMetadataDocument.setId(recordMetadata.getId()); + recordMetadataDocument.setAcl(recordMetadata.getAcl()); + recordMetadataDocument.setAncestry(recordMetadata.getAncestry()); + recordMetadataDocument.setCreateTime(recordMetadata.getCreateTime()); + recordMetadataDocument.setModifyTime(recordMetadata.getModifyTime()); + recordMetadataDocument.setGcsVersionPaths(recordMetadata.getGcsVersionPaths()); + recordMetadataDocument.setKind(recordMetadata.getKind()); + recordMetadataDocument.setLegal(recordMetadata.getLegal()); + recordMetadataDocument.setModifyUser(recordMetadata.getModifyUser()); + recordMetadataDocument.setStatus(recordMetadata.getStatus()); + recordMetadataDocument.setUser(recordMetadata.getUser()); + + return recordMetadataDocument; + } + + private RecordMetadata convertToRecordMetadata(RecordMetadataDocument recordMetadataDocument) { + if (Objects.isNull(recordMetadataDocument)) { + return null; + } + RecordMetadata recordMetadata = new RecordMetadata(); + recordMetadata.setId(recordMetadataDocument.getId()); + recordMetadata.setAcl(recordMetadataDocument.getAcl()); + recordMetadata.setAncestry(recordMetadataDocument.getAncestry()); + recordMetadata.setCreateTime(recordMetadataDocument.getCreateTime()); + recordMetadata.setModifyTime(recordMetadataDocument.getModifyTime()); + recordMetadata.setGcsVersionPaths(recordMetadataDocument.getGcsVersionPaths()); + recordMetadata.setKind(recordMetadataDocument.getKind()); + recordMetadata.setLegal(recordMetadataDocument.getLegal()); + recordMetadata.setModifyUser(recordMetadataDocument.getModifyUser()); + recordMetadata.setStatus(recordMetadataDocument.getStatus()); + recordMetadata.setUser(recordMetadataDocument.getUser()); + + return recordMetadata; + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java new file mode 100644 index 00000000..b9ba1012 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java @@ -0,0 +1,73 @@ +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.Objects; +import org.apache.http.HttpStatus; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository; +import org.opengroup.osdu.storage.provider.reference.model.SchemaDocument; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class SchemaRepositoryImpl implements ISchemaRepository { + + private static final Logger logger = LoggerFactory.getLogger(SchemaRepositoryImpl.class); + public static final String SCHEMA_STORAGE = "SchemaStorage"; + public static final String RECORD_STORAGE = "StorageRecord"; + public static final String SCHEMA_DATABASE = "schema"; + + @Autowired + private MongoDdmsClient mongoClient; + + @Override + public void add(Schema schema, String user) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + String kind = schema.getKind(); + FindIterable results = collection.find(eq("kind", kind)); + if (Objects.nonNull(results) && Objects.nonNull(results.first())) { + throw new IllegalArgumentException("Schema " + kind + " already exist. Can't create again."); + } + SchemaDocument schemaDocument = new SchemaDocument(schema, user); + collection.insertOne(Document.parse(new Gson().toJson(schemaDocument))); + } + + @Override + public Schema get(String kind) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + Document record = (Document) collection.find(eq("kind", kind)).first(); + if (Objects.isNull(record)) { + throw new AppException( + HttpStatus.SC_NOT_FOUND, "Not found", + String.format("Schema with id %s does not exist.", kind)); + } + SchemaDocument schemaDocument = new Gson().fromJson(record.toJson(), SchemaDocument.class); + return convertToSchemaEntity(schemaDocument); + } + + @Override + public void delete(String kind) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + collection.deleteOne(eq("kind", kind)); + } + + private Schema convertToSchemaEntity(SchemaDocument schemaDocument) { + Schema schema = new Schema(); + schema.setKind(schemaDocument.getKind()); + schema.setExt(schemaDocument.getExtension()); + schema.setSchema(schemaDocument.getSchema()); + return schema; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java new file mode 100644 index 00000000..2482333c --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java @@ -0,0 +1,18 @@ +package org.opengroup.osdu.storage.provider.reference.security; + +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity http) throws Exception { + http + .httpBasic().disable() + .csrf().disable(); + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java new file mode 100644 index 00000000..15300d52 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java @@ -0,0 +1,25 @@ +package org.opengroup.osdu.storage.provider.reference.security; + +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +public class WhoamiController { + + @RequestMapping(value = {"/", "/whoami"}) + @ResponseBody + public String whoami() { + final Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + String userName = auth.getName(); + String roles = String.valueOf(auth.getAuthorities()); + String details = String.valueOf(auth.getPrincipal()); + + return "user: " + userName + "
" + + "roles: " + roles + "
" + + "details: " + details; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java new file mode 100644 index 00000000..56fa7d97 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java @@ -0,0 +1,37 @@ +package org.opengroup.osdu.storage.provider.reference.service; + +import static java.util.Collections.singletonList; + +import javax.inject.Inject; +import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; +import org.opengroup.osdu.storage.logging.StorageAuditLogger; +import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; +import org.opengroup.osdu.storage.service.BatchServiceImpl; +import org.springframework.stereotype.Service; + +@Service +public class BatchServiceReferenceImpl extends BatchServiceImpl { + + @Inject + private StorageAuditLogger auditLogger; + + @Inject + private IQueryRepository queryRepository; + + @Override + public DatastoreQueryResult getAllKinds(String cursor, Integer limit) { + DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); + this.auditLogger.readAllKindsSuccess(result.getResults()); + return result; + } + + @Override + public DatastoreQueryResult getAllRecords(String cursor, String kind, Integer limit) { + DatastoreQueryResult result = this.queryRepository.getAllRecordIdsFromKind(kind, limit, cursor); + if (!result.getResults().isEmpty()) { + this.auditLogger.readAllRecordsOfGivenKindSuccess(singletonList(kind)); + } + return result; + } + +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java new file mode 100644 index 00000000..0147bc9a --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java @@ -0,0 +1,39 @@ +package org.opengroup.osdu.storage.provider.reference.service; + +import com.google.gson.Gson; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.storage.PubSubInfo; +import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; +import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MessageBusImpl implements IMessageBus { + + @Autowired + private IMessageFactory messageQueue; + + public void publishMessage(DpsHeaders headers, PubSubInfo... messages) { + final int BATCH_SIZE = 50; + Map message = new HashMap<>(); + Gson gson = new Gson(); + + for (int i = 0; i < messages.length; i += BATCH_SIZE) { + PubSubInfo[] batch = Arrays + .copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE)); + + String json = gson.toJson(batch); + message.put("data", json); + message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + + messageQueue.sendMessage(gson.toJson(message)); + } + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java new file mode 100644 index 00000000..fb35c3a4 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java @@ -0,0 +1,68 @@ +package org.opengroup.osdu.storage.provider.reference.util; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class MongoClientHandler { + + private static final Logger LOG = LoggerFactory.getLogger(MongoClientHandler.class); + private static final String MONGO_PREFIX = "mongodb://"; + private static final String MONGO_OPTIONS = "retryWrites=true&w=majority"; + + private com.mongodb.client.MongoClient mongoClient = null; + + @Value("${mongo.db.url:#{null}}") + private String dbUrl; + + @Value("${mongo.db.apikey:#{null}}") + private String apiKey; + + @Value("${mongo.db.user:#{null}}") + private String dbUser; + + @Value("${mongo.db.password:#{null}}") + private String dbPassword; + + private MongoClient getOrInitMongoClient() throws RuntimeException { + if (mongoClient != null) { + return mongoClient; + } + + final String connectionString = String.format("%s%s:%s@%s/?%s", + MONGO_PREFIX, + dbUser, + dbPassword, + dbUrl, + MONGO_OPTIONS); + ConnectionString connString = new ConnectionString(connectionString); + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connString) + .retryWrites(true) + .build(); + try { + mongoClient = MongoClients.create(settings); + } catch (Exception ex) { + LOG.error("Error connecting MongoDB", ex); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB", + ex.getMessage(), ex); + } + return mongoClient; + } + + public MongoClient getMongoClient() { + if (mongoClient == null) { + getOrInitMongoClient(); + } + return mongoClient; + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java new file mode 100644 index 00000000..ae5153b9 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java @@ -0,0 +1,14 @@ +package org.opengroup.osdu.storage.provider.reference.util; + +import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { + @Override + public String getIdToken(String tenantName){ + return "dont-have-one"; + } +} diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties new file mode 100644 index 00000000..9d979d6e --- /dev/null +++ b/provider/storage-reference/src/main/resources/application.properties @@ -0,0 +1,17 @@ +LOG_PREFIX=storage + +server.servlet.contextPath=/api/storage/v2/ +logging.level.org.springframework.web=DEBUG +server.port=8080 +JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M +JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 + +AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 +LEGALTAG_API=https://os-legal-ibm/api/legal/v1 + +mongo.db.url={MONGO_DB_URL} +mongo.db.user={MONGO_DB_USER} +mongo.db.password={MONGO_DB_PASSWORD} + +#amqp://guest:guest@127.0.0.1:5672/%2F by default +mb.rabbitmq.uri={RABBITMQ_URI} \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java new file mode 100644 index 00000000..d6e1fcc5 --- /dev/null +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java @@ -0,0 +1,16 @@ +package org.opengroup.osdu.storage.provider.reference.context; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class ProviderSpringContextTest { + + @Test + public void load_spring_context(){ + + } +} diff --git a/provider/storage-reference/src/test/resources/application.properties b/provider/storage-reference/src/test/resources/application.properties new file mode 100644 index 00000000..1f0fc86a --- /dev/null +++ b/provider/storage-reference/src/test/resources/application.properties @@ -0,0 +1,17 @@ +LOG_PREFIX=storage + +server.servlet.contextPath=/api/storage/v2/ +logging.level.org.springframework.web=DEBUG +server.port=8080 +JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M +JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 + +AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 +LEGALTAG_API=https://os-legal-ibm/api/legal/v1 + +mongo.db.url=test +mongo.db.user=test +mongo.db.password=test + +#amqp://guest:guest@127.0.0.1:5672/%2F by default +mb.rabbitmq.uri=amqp://guest:guest@test:5672/%2F \ No newline at end of file -- GitLab From e53fbb3afd12b818ba82d8c718b505d707cad9e6 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Tue, 27 Oct 2020 10:15:33 +0400 Subject: [PATCH 04/44] GONRG-1019: minio for Storage --- provider/storage-reference/pom.xml | 6 +- .../provider/reference/CloudStorageImpl.java | 198 +++++++++++------- .../factory/CloudObjectStorageFactory.java | 45 ++++ .../src/main/resources/application.properties | 10 +- .../reference/CloudStorageImplTest.java | 100 +++++++++ 5 files changed, 284 insertions(+), 75 deletions(-) create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java create mode 100644 provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java diff --git a/provider/storage-reference/pom.xml b/provider/storage-reference/pom.xml index ebade6bc..778959bc 100644 --- a/provider/storage-reference/pom.xml +++ b/provider/storage-reference/pom.xml @@ -84,7 +84,11 @@ amqp-client 5.7.3 - + + io.minio + minio + 7.1.4 + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java index b5db09b2..3c2f1db9 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java @@ -1,39 +1,89 @@ package org.opengroup.osdu.storage.provider.reference; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import org.opengroup.osdu.core.common.model.storage.*; +import com.google.gson.GsonBuilder; +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.RecordData; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordProcessing; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.opengroup.osdu.core.common.model.storage.TransferInfo; import org.opengroup.osdu.core.common.util.Crc32c; import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; import org.apache.http.HttpStatus; +import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.MediaType; import org.springframework.stereotype.Repository; import java.nio.charset.StandardCharsets; -import java.util.*; import static org.apache.commons.codec.binary.Base64.encodeBase64; @Repository public class CloudStorageImpl implements ICloudStorage { - private static List memList = new ArrayList<>(); + + @Value("${minio.bucket.record.name}") + private String recordBucketName; + + @Autowired + private CloudObjectStorageFactory factory; + + @Autowired + private JaxRsDpsLog log; + + private MinioClient minioClient; + + @PostConstruct + public void init() { + minioClient = factory.getClient(); + } @Override - public void write(RecordProcessing... recordsProcessing){ + public void write(RecordProcessing... recordsProcessing) { + Gson gson = new GsonBuilder().serializeNulls().create(); for (RecordProcessing rp: recordsProcessing) { - memList.add(rp); + Map headers = new HashMap<>(); + headers.put("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE); + headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); + + String content = gson.toJson(rp.getRecordData()); + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + String itemName = getItemName(rp.getRecordMetadata()).replace(":", "-"); + try { + minioClient.putObject( + PutObjectArgs.builder().bucket(recordBucketName).object(itemName).stream( + new ByteArrayInputStream(bytes), bytes.length, -1) + .headers(headers) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, + "Failed to write new record.", e.getMessage()); + } } } @Override - public Map getHash(Collection records) - { + public Map getHash(Collection records) { Gson gson = new Gson(); Map hashes = new HashMap<>(); - for (RecordMetadata rm : records) - { - String jsonData = this.read(rm, rm.getLatestVersion(), false); + for (RecordMetadata rm : records) { + String jsonData = read(rm, rm.getLatestVersion(), false); RecordData data = gson.fromJson(jsonData, RecordData.class); String hash = getHash(data); @@ -70,41 +120,45 @@ public class CloudStorageImpl implements ICloudStorage { return newHash; } + private String getItemName(RecordMetadata record) { + return record.getVersionPath(record.getLatestVersion()); + } + + private String getItemName(RecordMetadata record, Long version) { + return record.getVersionPath(version); + } + + public void setRecordBucketName(String recordBucketName) { + this.recordBucketName = recordBucketName; + } + @Override - public void delete(RecordMetadata record) - { - Iterator it = memList.iterator(); - while(it.hasNext()) - { - RecordProcessing rp = it.next(); - if (rp.getRecordMetadata().getId().equals(record.getId())) - { - it.remove(); - } + public void delete(RecordMetadata record) { + String itemName = getItemName(record).replace(":", "-"); + try { + minioClient.removeObject( + RemoveObjectArgs.builder().bucket(recordBucketName).object(itemName).build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete record.", e.getMessage()); } } @Override public void deleteVersion(RecordMetadata record, Long version) { - //TODO: Check if this is correct - - Iterator it = memList.iterator(); - while(it.hasNext()) - { - RecordProcessing rp = it.next(); - if (rp.getRecordMetadata().getId().equals(record.getId())) { - for (String path : rp.getRecordMetadata().getGcsVersionPaths()) { - if (path.contains(version.toString())) { - it.remove(); - } - } + String itemName = getItemName(record, version).replace(":", "-"); + try { + if (!record.hasVersion()) { + log.warning(String.format("Record %s does not have versions available", record.getId())); } + minioClient.removeObject( + RemoveObjectArgs.builder().bucket(recordBucketName).object(itemName).build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete version.", e.getMessage()); } } @Override - public boolean hasAccess(RecordMetadata... records) - { + public boolean hasAccess(RecordMetadata... records) { for (RecordMetadata record : records) { if (!record.getStatus().equals(RecordState.active)) @@ -114,51 +168,51 @@ public class CloudStorageImpl implements ICloudStorage { } @Override - public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) - { - for (RecordProcessing rp : memList) - { - RecordMetadata rmd = rp.getRecordMetadata(); - if (rmd.getId().equals(record.getId())) - { - for (String path: rmd.getGcsVersionPaths()) - { - if (path.contains(version.toString())) { - try { - String result = new ObjectMapper().writeValueAsString(rp.getRecordData()); - return result; - } catch (JsonProcessingException je) - { - throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", je.getMessage()); - } - } - } + public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) { + String itemName = getItemName(record, version).replace(":", "-"); + String msg = String.format("Record with id '%s' does not exist, version: %s", record.getId(), version); + InputStream stream; + try { + stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(recordBucketName) + .object(itemName) + .build()); + if (stream == null) { + log.warning(msg); + } else { + return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", e.getMessage()); } - String msg = String.format("Record with id '%s' does not exist", record.getId()); throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg); } @Override - public Map read(Map objects) - { - Map map = new HashMap<>(); - for (RecordProcessing rp : memList) - { - RecordMetadata rmd = rp.getRecordMetadata(); - String id = rmd.getId(); - if (objects.containsKey(id)) - { - try { - String result = new ObjectMapper().writeValueAsString(rp.getRecordData()); - map.put(id, result); - } catch (JsonProcessingException je) - { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Record to json failed", je.getMessage()); + public Map read(Map objects) { + // key -> record id + // value -> record version path + Map map = new HashMap<>(); + for (Map.Entry record : objects.entrySet()) { + String[] tokens = record.getValue().split("/"); + String key = tokens[tokens.length - 2]; + try { + InputStream stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(recordBucketName) + .object(record.getValue().replace(":", "-")) + .build()); + if (stream != null) { + String result = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + map.put(key, result); } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", e.getMessage()); } } return map; } - } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java new file mode 100644 index 00000000..3f2e6ad2 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java @@ -0,0 +1,45 @@ +package org.opengroup.osdu.storage.provider.reference.factory; + +import io.minio.MinioClient; +import javax.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component +@Lazy +public class CloudObjectStorageFactory { + private static final Logger logger = LoggerFactory.getLogger(CloudObjectStorageFactory.class); + + @Value("${minio.endpoint_url}") + private String endpointURL; + @Value("${minio.access_key}") + private String accessKey; + @Value("${minio.secret_key}") + private String secretKey; + @Value("${minio.region:us-east-1}") + private String region; + @Value("${minio.prefix:local-dev}") + private String bucketNamePrefix; + + private MinioClient minioClient; + + private String bucketName; + + public CloudObjectStorageFactory() { } + + @PostConstruct + public void init() { + minioClient = MinioClient.builder() + .endpoint(endpointURL) + .credentials(accessKey, secretKey) + .region(region).build(); + logger.info("Minio client initialized"); + } + + public MinioClient getClient() { + return this.minioClient; + } +} diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties index 9d979d6e..5906446c 100644 --- a/provider/storage-reference/src/main/resources/application.properties +++ b/provider/storage-reference/src/main/resources/application.properties @@ -1,5 +1,4 @@ LOG_PREFIX=storage - server.servlet.contextPath=/api/storage/v2/ logging.level.org.springframework.web=DEBUG server.port=8080 @@ -14,4 +13,11 @@ mongo.db.user={MONGO_DB_USER} mongo.db.password={MONGO_DB_PASSWORD} #amqp://guest:guest@127.0.0.1:5672/%2F by default -mb.rabbitmq.uri={RABBITMQ_URI} \ No newline at end of file +mb.rabbitmq.uri={RABBITMQ_URI} + +minio.endpoint_url=http://127.0.0.1:9000 +minio.access_key=adminadmin +minio.secret_key=adminadmin +minio.region=admin +minio.prefix=local-dev +minio.bucket.record.name=record-bucket \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java new file mode 100644 index 00000000..1ad3548f --- /dev/null +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java @@ -0,0 +1,100 @@ +package org.opengroup.osdu.storage.provider.reference; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; +import io.minio.MinioClient; +import java.util.HashMap; +import java.util.Map; +import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.Record; +import org.opengroup.osdu.core.common.model.storage.RecordData; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordProcessing; +import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; + +@RunWith(MockitoJUnitRunner.class) +public class CloudStorageImplTest extends TestCase { + + @Mock + private CloudObjectStorageFactory factory; + + @Mock + private MinioClient minioClient; + + @Mock + private JaxRsDpsLog log; + + @InjectMocks + private CloudStorageImpl cloudStorage; + + private RecordProcessing[] recordProcessings = new RecordProcessing[1]; + private RecordProcessing recordProcessing; + private RecordMetadata recordMetadata; + + @Before + public void setup() { + initMocks(this); + + recordMetadata = new RecordMetadata(); + recordMetadata.setKind("test-record-id"); + recordMetadata.setId("test-record-id"); + recordMetadata.addGcsPath(1); + recordMetadata.addGcsPath(2); + + recordProcessing = new RecordProcessing(); + recordProcessing.setRecordMetadata(recordMetadata); + + Record record = new Record(); + record.setId("test-record-id"); + Map data = new HashMap<>(); + data.put("test-data", new Object()); + record.setData(data); + + RecordData recordData = new RecordData(record); + recordProcessing.setRecordData(recordData); + recordProcessings[0] = recordProcessing; + cloudStorage.setRecordBucketName("record-bucket"); + } + + @Test + public void write_test() throws Exception { + cloudStorage.write(recordProcessings); + verify(minioClient, times(1)).putObject(any()); + } + + @Test + public void delete_test() throws Exception { + cloudStorage.delete(recordMetadata); + verify(minioClient, times(1)).removeObject(any()); + } + + @Test + public void deleteVersion_test() throws Exception { + cloudStorage.deleteVersion(recordMetadata, (long) 1); + verify(minioClient, times(1)).removeObject(any()); + } + + @Test(expected = AppException.class) + public void read_with_version_test() throws Exception { + cloudStorage.read(recordMetadata, (long) 1, false); + verify(minioClient, times(1)).getObject(any()); + } + + @Test + public void read_test() throws Exception { + Map map = new HashMap<>(); + map.put("common:welldb:1", "opendes:ds:mytest1:1.0.0/common:welldb:123456/1603618609515093"); + cloudStorage.read(map); + verify(minioClient, times(1)).getObject(any()); + } +} \ No newline at end of file -- GitLab From d951603e78cf07dddc779c39d5bc86397cbc130c Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Wed, 28 Oct 2020 17:42:13 +0400 Subject: [PATCH 05/44] GONRG-993 Updated all pom files. --- .mvn/community-maven.settings.xml | 26 ++++++++++++++++++---- pom.xml | 35 +++++++++++++++--------------- provider/storage-aws/pom.xml | 20 +---------------- provider/storage-azure/pom.xml | 19 +--------------- provider/storage-byoc/pom.xml | 20 +---------------- provider/storage-gcp/pom.xml | 19 +--------------- provider/storage-ibm/pom.xml | 20 +---------------- storage-core/pom.xml | 18 --------------- testing/pom.xml | 12 +++++----- testing/storage-test-aws/pom.xml | 25 ++++++--------------- testing/storage-test-azure/pom.xml | 25 ++++++--------------- testing/storage-test-core/pom.xml | 24 ++++++-------------- testing/storage-test-gcp/pom.xml | 26 ++++++---------------- testing/storage-test-ibm/pom.xml | 25 ++++++--------------- 14 files changed, 86 insertions(+), 228 deletions(-) diff --git a/.mvn/community-maven.settings.xml b/.mvn/community-maven.settings.xml index 573c471b..4e70f459 100644 --- a/.mvn/community-maven.settings.xml +++ b/.mvn/community-maven.settings.xml @@ -7,14 +7,20 @@ true - community-maven-via-job-token + community-maven-repo + community-maven-via-job-token + community-maven-via-job-token + + https://community.opengroup.org/api/v4/groups/17/-/packages/maven + https://community.opengroup.org/api/v4/projects/44/packages/maven + https://community.opengroup.org/api/v4/projects/44/packages/maven + presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token + based authentication --> GitLab-Authenticate-With-Private-Token @@ -22,7 +28,13 @@ - community-maven-via-private-token + community-maven-repo + community-maven-via-private-token + community-maven-via-private-token + + https://community.opengroup.org/api/v4/groups/17/-/packages/maven + https://community.opengroup.org/api/v4/projects/44/packages/maven + https://community.opengroup.org/api/v4/projects/44/packages/maven @@ -61,5 +73,11 @@ AZURE + + + os-core + slb-des-ext-collaboration + ${VSTS_FEED_TOKEN} + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3a9c6ad6..cf1357a5 100644 --- a/pom.xml +++ b/pom.xml @@ -105,13 +105,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - storage-core provider/storage-gcp @@ -121,15 +114,23 @@ provider/storage-ibm - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - + + + ${repo.releases.id} + ${repo.releases.url} + + + + + + ${publish.releases.id} + ${publish.releases.url} + + + ${publish.snapshots.id} + ${publish.snapshots.url} + + + diff --git a/provider/storage-aws/pom.xml b/provider/storage-aws/pom.xml index 96e20550..018cd9a8 100644 --- a/provider/storage-aws/pom.xml +++ b/provider/storage-aws/pom.xml @@ -22,7 +22,7 @@ os-storage org.opengroup.osdu 0.0.5-SNAPSHOT - ../.. + ../../pom.xml 4.0.0 @@ -167,22 +167,4 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/provider/storage-azure/pom.xml b/provider/storage-azure/pom.xml index fc74d443..78c770c7 100644 --- a/provider/storage-azure/pom.xml +++ b/provider/storage-azure/pom.xml @@ -22,7 +22,7 @@ os-storage org.opengroup.osdu 0.0.5-SNAPSHOT - ../../ + ../../pom.xml 4.0.0 storage-azure @@ -140,23 +140,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - diff --git a/provider/storage-byoc/pom.xml b/provider/storage-byoc/pom.xml index 85b0a2c7..5cd01fd9 100644 --- a/provider/storage-byoc/pom.xml +++ b/provider/storage-byoc/pom.xml @@ -22,7 +22,7 @@ os-storage org.opengroup.osdu 0.0.5-SNAPSHOT - ../.. + ../../pom.xml 4.0.0 @@ -70,24 +70,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 72b002af..bd64d7e2 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -28,7 +28,7 @@ org.opengroup.osdu os-storage 0.0.5-SNAPSHOT - ../../ + ../../pom.xml @@ -90,23 +90,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - diff --git a/provider/storage-ibm/pom.xml b/provider/storage-ibm/pom.xml index 68940f1a..336661d7 100644 --- a/provider/storage-ibm/pom.xml +++ b/provider/storage-ibm/pom.xml @@ -19,7 +19,7 @@ os-storage org.opengroup.osdu 0.0.5-SNAPSHOT - ../.. + ../../pom.xml 4.0.0 @@ -74,24 +74,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/storage-core/pom.xml b/storage-core/pom.xml index c63da60a..95535949 100644 --- a/storage-core/pom.xml +++ b/storage-core/pom.xml @@ -158,24 +158,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/testing/pom.xml b/testing/pom.xml index 75d0e6c1..86dc8393 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -41,19 +41,19 @@ - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven + ${repo.releases.id} + ${repo.releases.url} - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven + ${publish.releases.id} + ${publish.releases.url} - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven + ${publish.snapshots.id} + ${publish.snapshots.url} diff --git a/testing/storage-test-aws/pom.xml b/testing/storage-test-aws/pom.xml index 6e7dcb93..354c6e46 100644 --- a/testing/storage-test-aws/pom.xml +++ b/testing/storage-test-aws/pom.xml @@ -17,6 +17,13 @@ + + os-storage-testing + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../pom.xml + + 4.0.0 org.opengroup.osdu.storage @@ -103,24 +110,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/testing/storage-test-azure/pom.xml b/testing/storage-test-azure/pom.xml index 1b0c58cb..1056eaad 100644 --- a/testing/storage-test-azure/pom.xml +++ b/testing/storage-test-azure/pom.xml @@ -18,6 +18,13 @@ + + os-storage-testing + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../pom.xml + + 4.0.0 org.opengroup.osdu.storage @@ -95,24 +102,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - diff --git a/testing/storage-test-core/pom.xml b/testing/storage-test-core/pom.xml index 12945865..c6ccc484 100644 --- a/testing/storage-test-core/pom.xml +++ b/testing/storage-test-core/pom.xml @@ -17,6 +17,13 @@ + + os-storage-testing + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../pom.xml + + 4.0.0 org.opengroup.osdu.storage storage-test-core @@ -64,23 +71,6 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - diff --git a/testing/storage-test-gcp/pom.xml b/testing/storage-test-gcp/pom.xml index 01c1b5a0..e2be355c 100644 --- a/testing/storage-test-gcp/pom.xml +++ b/testing/storage-test-gcp/pom.xml @@ -18,6 +18,13 @@ + + os-storage-testing + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../pom.xml + + 4.0.0 org.opengroup.osdu.storage @@ -127,25 +134,6 @@ - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - test diff --git a/testing/storage-test-ibm/pom.xml b/testing/storage-test-ibm/pom.xml index e32edf73..10a26e9f 100644 --- a/testing/storage-test-ibm/pom.xml +++ b/testing/storage-test-ibm/pom.xml @@ -17,6 +17,13 @@ + + os-storage-testing + org.opengroup.osdu + 0.0.5-SNAPSHOT + ../pom.xml + + 4.0.0 org.opengroup.osdu.storage @@ -99,22 +106,4 @@ - - - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - - - - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/44/packages/maven - - - -- GitLab From 43053e0684f09a055eab1ef34224b1dbe235f262 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Mon, 2 Nov 2020 13:38:30 +0400 Subject: [PATCH 06/44] GONRG-1004: Deploy Storage into Anthos Service --- provider/storage-reference/Dockerfile | 5 + provider/storage-reference/docker-compose.yml | 12 ++ .../deployment-os-storage-service.yml | 132 ++++++++++++++++++ .../reference/di/TenantFactoryImpl.java | 7 +- skaffold.yaml | 12 ++ 5 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 provider/storage-reference/Dockerfile create mode 100644 provider/storage-reference/docker-compose.yml create mode 100644 provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml create mode 100644 skaffold.yaml diff --git a/provider/storage-reference/Dockerfile b/provider/storage-reference/Dockerfile new file mode 100644 index 00000000..ff9eb802 --- /dev/null +++ b/provider/storage-reference/Dockerfile @@ -0,0 +1,5 @@ +FROM openjdk:8-slim +WORKDIR /app +COPY target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar storage-reference.jar +# Run the web service on container startup. +CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/storage-reference.jar diff --git a/provider/storage-reference/docker-compose.yml b/provider/storage-reference/docker-compose.yml new file mode 100644 index 00000000..213fdb3e --- /dev/null +++ b/provider/storage-reference/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3" +services: + os-storage-app: + build: + args: + JAR_FILE: target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar + context: "" + dockerfile: ../Dockerfile + image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference + ports: + - "8080:8080" + diff --git a/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml new file mode 100644 index 00000000..9ff958c6 --- /dev/null +++ b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml @@ -0,0 +1,132 @@ +apiVersion: v1 +data: + AUTHORIZE_API: ${AUTHORIZE_API} + MONGO_DB_URL: ${MONGO_DB_URL} + MONGO_DB_USER: ${MONGO_DB_USER} + MONGO_DB_NAME: ${MONGO_DB_NAME} + REGION: ${REGION} + LEGALTAG_API: ${LEGALTAG_API} + org.opengroup.osdu.storage.disableAuth: "true" +kind: ConfigMap +metadata: + labels: + app: storage-reference + name: storage-config + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + generateName: storage-reference-anthos + labels: + app: storage-reference + name: storage-reference + namespace: default +spec: + selector: + matchLabels: + app: storage-reference + replicas: 1 + template: + metadata: + labels: + app: storage-reference + spec: + containers: + - env: + - name: AUTHORIZE_API + valueFrom: + configMapKeyRef: + key: AUTHORIZE_API + name: storage-config + - name: MONGO_DB_URL + valueFrom: + configMapKeyRef: + key: MONGO_DB_URL + name: storage-config + - name: MONGO_DB_USER + valueFrom: + configMapKeyRef: + key: MONGO_DB_USER + name: storage-config + - name: MONGO_DB_PASSWORD + valueFrom: + secretKeyRef: + name: storage-secret + key: mongo.db.password + - name: MONGO_DB_NAME + valueFrom: + configMapKeyRef: + key: MONGO_DB_NAME + name: storage-config + - name: MB_RABBITMQ_URI + valueFrom: + secretKeyRef: + name: storage-secret + key: mb.rabbitmq.uri + - name: REGION + valueFrom: + configMapKeyRef: + key: REGION + name: storage-config + - name: org.opengroup.osdu.storage.disableAuth + valueFrom: + configMapKeyRef: + key: org.opengroup.osdu.storage.disableAuth + name: storage-config + - name: MINIO_URL + valueFrom: + secretKeyRef: + key: minio.enpoint_url + name: storage-secret + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: minio.access_key + name: storage-secret + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: minio.secret_key + name: storage-secret + - name: MINIO_REGION + valueFrom: + secretKeyRef: + key: minio.region + name: storage-secret + - name: MINIO_BUCKET_RECORD_NAME + valueFrom: + secretKeyRef: + key: minio.bucket.record.name + name: storage-secret + image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference:9a1d20e-dirty + name: storage-reference +--- +apiVersion: v1 +kind: Service +metadata: + name: storage-reference + namespace: default +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + selector: + app: storage-reference + type: LoadBalancer +--- +apiVersion: v1 +data: + mongo.db.password: ${MONGO_DB_PASSWORD} + mb.rabbitmq.uri: ${MB_RABBITMQ_URI} + minio.enpoint_url: ${MINIO_URL} + minio.access_key: ${MINIO_ACCESS_KEY} + minio.secret_key: ${MINIO_SECRET_KEY} + minio.region: ${MINIO_REGION} + minio.bucket.record.name: ${MINIO_RECORD_BUCKET_NAME} +kind: Secret +metadata: + name: storage-secret + namespace: default +type: Opaque diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java index 0a28547e..45e68565 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java @@ -1,7 +1,5 @@ package org.opengroup.osdu.storage.provider.reference.di; -import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; - import com.google.gson.Gson; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; @@ -24,7 +22,8 @@ import org.springframework.stereotype.Component; public class TenantFactoryImpl implements ITenantFactory { private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); - public static final String TENANT_INFO = "TenantInfo"; + public static final String TENANT_INFO = "tenantinfo"; + public static final String MAIN_DATABASE = "main"; @Autowired private MongoDdmsClient mongoClient; @@ -63,7 +62,7 @@ public class TenantFactoryImpl implements ITenantFactory { private void initTenants() { this.tenants = new HashMap<>(); MongoCollection mongoCollection = mongoClient - .getMongoCollection(SCHEMA_DATABASE, TENANT_INFO); + .getMongoCollection(MAIN_DATABASE, TENANT_INFO); FindIterable results = mongoCollection.find(); if (Objects.isNull(results) && Objects.isNull(results.first())) { LOG.error(String.format("Collection \'%s\' is empty.", results)); diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000..4e5375a8 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,12 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: storage-reference +build: + artifacts: + - image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference + context: ./provider/storage-reference +deploy: + kubectl: + manifests: + - ./provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml \ No newline at end of file -- GitLab From 1c04aa30a16beb1ac6513e554389ebf387fd1a0d Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 27 Nov 2020 00:01:36 +0400 Subject: [PATCH 07/44] Bugfix for tenant info --- .../reference/di/TenantFactoryImpl.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java index 45e68565..6473d132 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java @@ -1,17 +1,20 @@ package org.opengroup.osdu.storage.provider.reference.di; +import static java.util.Objects.isNull; + import com.google.gson.Gson; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Objects; import org.bson.Document; +import org.bson.types.ObjectId; import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.storage.provider.reference.model.TenantInfoDocument; import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,21 +67,16 @@ public class TenantFactoryImpl implements ITenantFactory { MongoCollection mongoCollection = mongoClient .getMongoCollection(MAIN_DATABASE, TENANT_INFO); FindIterable results = mongoCollection.find(); - if (Objects.isNull(results) && Objects.isNull(results.first())) { + if (isNull(results) || isNull(results.first())) { LOG.error(String.format("Collection \'%s\' is empty.", results)); } for (Document document : results) { - TenantInfoDocument tenantInfoDocument = new Gson() - .fromJson(document.toJson(), TenantInfoDocument.class); - TenantInfo tenantInfo = convertToTenantInfo(tenantInfoDocument); + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class); + ObjectId id = (ObjectId) document.get("_id"); + tenantInfo.setId((long) id.getCounter()); + tenantInfo.setCrmAccountIds((ArrayList) document.get("crmAccountID")); this.tenants.put(tenantInfo.getName(), tenantInfo); } } - - private TenantInfo convertToTenantInfo(TenantInfoDocument tenantInfoDocument) { - TenantInfo tenantInfo = new TenantInfo(); - tenantInfo.setName(tenantInfoDocument.getId()); - return tenantInfo; - } } -- GitLab From 2d5a03e5a10cf2db20cd3adb7d3eaed8ee44b61a Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 27 Nov 2020 19:42:42 +0400 Subject: [PATCH 08/44] Fix mongo issues and cache --- .../reference/app/StorageReferenceApplication.java | 3 ++- .../provider/reference/cache/GroupCache.java | 14 +++++++++----- .../reference/util/MongoClientHandler.java | 2 +- .../src/main/resources/application.properties | 6 +++++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java index 595e3ebd..4767155b 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java @@ -2,9 +2,10 @@ package org.opengroup.osdu.storage.provider.reference.app; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import org.springframework.context.annotation.ComponentScan; -@SpringBootApplication +@SpringBootApplication(exclude={MongoAutoConfiguration.class}) @ComponentScan({"org.opengroup.osdu"}) public class StorageReferenceApplication { diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java index c94499f5..291d512a 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java @@ -1,13 +1,17 @@ package org.opengroup.osdu.storage.provider.reference.cache; -import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.entitlements.Groups; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component -public class GroupCache extends VmCache { +public class GroupCache extends RedisCache { - public GroupCache() { - super(5 * 60, 1000); - } + public GroupCache( + @Value("${gcp.redis.host}") final String redisHost, + @Value("${gcp.redis.port}") final Integer redisPort, + @Value("${gcp.redis.exp.time}") final Integer expTimeSec) { + super(redisHost, redisPort, expTimeSec, String.class, Groups.class); + } } \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java index fb35c3a4..976463c8 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java @@ -16,7 +16,7 @@ public class MongoClientHandler { private static final Logger LOG = LoggerFactory.getLogger(MongoClientHandler.class); private static final String MONGO_PREFIX = "mongodb://"; - private static final String MONGO_OPTIONS = "retryWrites=true&w=majority"; + private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; private com.mongodb.client.MongoClient mongoClient = null; diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties index 5906446c..fd0e55a7 100644 --- a/provider/storage-reference/src/main/resources/application.properties +++ b/provider/storage-reference/src/main/resources/application.properties @@ -20,4 +20,8 @@ minio.access_key=adminadmin minio.secret_key=adminadmin minio.region=admin minio.prefix=local-dev -minio.bucket.record.name=record-bucket \ No newline at end of file +minio.bucket.record.name=record-bucket + +gcp.redis.host={GCP_REDIS_HOST} +gcp.redis.port={GCP_REDIS_PORT} +gcp.redis.exp.time={GCP_REDIS_EXP_TIME} \ No newline at end of file -- GitLab From ce2e937cbc088c589722860d1876aafb7e5d9dbf Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Mon, 30 Nov 2020 11:54:39 +0400 Subject: [PATCH 09/44] Fix rabbit-que naming --- .../storage/provider/reference/di/RabbitMQFactoryImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java index 69bf9d38..69a1e4b3 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java @@ -38,8 +38,8 @@ public class RabbitMQFactoryImpl implements IMessageFactory { this.channel = conn.createChannel(); LOG.debug("RabbitMQ Channel was created."); for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { - channel.queueDeclare("os-storage-" + queue, false, false, false, null); - LOG.debug("Queue [os-storage-" + queue + "] was declared."); + channel.queueDeclare(queue, true, false, false, null); + LOG.debug("Queue [" + queue + "] was declared."); } } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { LOG.error(e.getMessage(), e); @@ -53,7 +53,7 @@ public class RabbitMQFactoryImpl implements IMessageFactory { @Override public void sendMessage(String queueName, String msg) { - String queueNameWithPrefix = "os-storage-" + queueName; + String queueNameWithPrefix = queueName; try { channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); -- GitLab From 9e61ac9d681ae9df29f5a8c8b7a980a6cb2139ec Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Mon, 30 Nov 2020 21:29:32 +0400 Subject: [PATCH 10/44] Bugfix for token in message --- .../osdu/storage/provider/reference/service/MessageBusImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java index 0147bc9a..390996e4 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java @@ -31,7 +31,7 @@ public class MessageBusImpl implements IMessageBus { message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); headers.addCorrelationIdIfMissing(); message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - + message.put(DpsHeaders.AUTHORIZATION,headers.getAuthorization()); messageQueue.sendMessage(gson.toJson(message)); } } -- GitLab From cf25e3fcbe56bb6fa809d89446e39f8ab4c1003d Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Tue, 1 Dec 2020 14:58:47 +0400 Subject: [PATCH 11/44] GONRG-1211 Update core common version with url normalization fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fb8b6f3b..cacca2c7 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 1.8 opendes UTF-8 - 0.3.18 + 0.3.21 2.11.2 4.1.51.Final 1.26 -- GitLab From 5963a9196e58dd80bd94ed4073f81065d7bce2fd Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 2 Dec 2020 14:30:12 +0400 Subject: [PATCH 12/44] Fill application properties with placeholders Ignore Spring context test, due to context loading dependencies on real connections --- .../src/main/resources/application.properties | 14 +++++++------- .../context/ProviderSpringContextTest.java | 3 ++- .../src/test/resources/application.properties | 19 +++++++++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties index fd0e55a7..0c536030 100644 --- a/provider/storage-reference/src/main/resources/application.properties +++ b/provider/storage-reference/src/main/resources/application.properties @@ -8,12 +8,12 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 LEGALTAG_API=https://os-legal-ibm/api/legal/v1 -mongo.db.url={MONGO_DB_URL} -mongo.db.user={MONGO_DB_USER} -mongo.db.password={MONGO_DB_PASSWORD} +mongo.db.url=localhost:27017 +mongo.db.user=admin +mongo.db.password=admin #amqp://guest:guest@127.0.0.1:5672/%2F by default -mb.rabbitmq.uri={RABBITMQ_URI} +mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F minio.endpoint_url=http://127.0.0.1:9000 minio.access_key=adminadmin @@ -22,6 +22,6 @@ minio.region=admin minio.prefix=local-dev minio.bucket.record.name=record-bucket -gcp.redis.host={GCP_REDIS_HOST} -gcp.redis.port={GCP_REDIS_PORT} -gcp.redis.exp.time={GCP_REDIS_EXP_TIME} \ No newline at end of file +gcp.redis.host=localhost +gcp.redis.port=6379 +gcp.redis.exp.time=10 \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java index d6e1fcc5..88c8c0d5 100644 --- a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java @@ -1,10 +1,11 @@ package org.opengroup.osdu.storage.provider.reference.context; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; - +@Ignore @RunWith(SpringRunner.class) @SpringBootTest public class ProviderSpringContextTest { diff --git a/provider/storage-reference/src/test/resources/application.properties b/provider/storage-reference/src/test/resources/application.properties index 1f0fc86a..122cf440 100644 --- a/provider/storage-reference/src/test/resources/application.properties +++ b/provider/storage-reference/src/test/resources/application.properties @@ -9,9 +9,20 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 LEGALTAG_API=https://os-legal-ibm/api/legal/v1 -mongo.db.url=test -mongo.db.user=test -mongo.db.password=test +mongo.db.url=localhost:27017 +mongo.db.user=admin +mongo.db.password=admin #amqp://guest:guest@127.0.0.1:5672/%2F by default -mb.rabbitmq.uri=amqp://guest:guest@test:5672/%2F \ No newline at end of file +mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F + +minio.endpoint_url=http://127.0.0.1:9000 +minio.access_key=adminadmin +minio.secret_key=adminadmin +minio.region=admin +minio.prefix=local-dev +minio.bucket.record.name=record-bucket + +gcp.redis.host=localhost +gcp.redis.port=6379 +gcp.redis.exp.time=10 \ No newline at end of file -- GitLab From 5e72a306a36150d45cff9e8bfa1e030274eab653 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 2 Dec 2020 16:04:09 +0400 Subject: [PATCH 13/44] Properties fix --- .../reference/factory/CloudObjectStorageFactory.java | 6 +++--- .../src/main/resources/application.properties | 6 +++--- .../src/test/resources/application.properties | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java index 3f2e6ad2..5a4dbcf8 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java @@ -13,11 +13,11 @@ import org.springframework.stereotype.Component; public class CloudObjectStorageFactory { private static final Logger logger = LoggerFactory.getLogger(CloudObjectStorageFactory.class); - @Value("${minio.endpoint_url}") + @Value("${minio.endpoint.url}") private String endpointURL; - @Value("${minio.access_key}") + @Value("${minio.access.key}") private String accessKey; - @Value("${minio.secret_key}") + @Value("${minio.secret.key}") private String secretKey; @Value("${minio.region:us-east-1}") private String region; diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties index 0c536030..d459d529 100644 --- a/provider/storage-reference/src/main/resources/application.properties +++ b/provider/storage-reference/src/main/resources/application.properties @@ -15,9 +15,9 @@ mongo.db.password=admin #amqp://guest:guest@127.0.0.1:5672/%2F by default mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F -minio.endpoint_url=http://127.0.0.1:9000 -minio.access_key=adminadmin -minio.secret_key=adminadmin +minio.endpoint.url=http://127.0.0.1:9000 +minio.access.key=adminadmin +minio.secret.key=adminadmin minio.region=admin minio.prefix=local-dev minio.bucket.record.name=record-bucket diff --git a/provider/storage-reference/src/test/resources/application.properties b/provider/storage-reference/src/test/resources/application.properties index 122cf440..7e2bae18 100644 --- a/provider/storage-reference/src/test/resources/application.properties +++ b/provider/storage-reference/src/test/resources/application.properties @@ -16,9 +16,9 @@ mongo.db.password=admin #amqp://guest:guest@127.0.0.1:5672/%2F by default mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F -minio.endpoint_url=http://127.0.0.1:9000 -minio.access_key=adminadmin -minio.secret_key=adminadmin +minio.endpoint.url=http://127.0.0.1:9000 +minio.access.key=adminadmin +minio.secret.key=adminadmin minio.region=admin minio.prefix=local-dev minio.bucket.record.name=record-bucket -- GitLab From dc8f357efa3bcfb4b2eb9d6ab11fccfd274025b0 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Thu, 3 Dec 2020 14:15:56 +0400 Subject: [PATCH 14/44] Creating new Publisher instance every time we want publish causing java.lang.RuntimeException: ManagedChannel allocation site --- .../storage/provider/gcp/GooglePubSub.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java index 518abe3b..994a9fdb 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java @@ -21,6 +21,7 @@ import com.google.protobuf.ByteString; import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.v1.PubsubMessage.Builder; +import java.util.Objects; import org.opengroup.osdu.core.common.model.storage.PubSubInfo; import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; import org.apache.http.HttpStatus; @@ -39,6 +40,8 @@ public class GooglePubSub implements IMessageBus { @Value("${PUBSUB_SEARCH_TOPIC}") public String PUBSUB_SEARCH_TOPIC; + private Publisher publisher; + private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder() .setTotalTimeout(Duration.ofSeconds(10)) @@ -56,17 +59,17 @@ public class GooglePubSub implements IMessageBus { @Override public void publishMessage(DpsHeaders headers, PubSubInfo... messages) { - Publisher publisher = null; - - try { - publisher = Publisher.newBuilder( + if(Objects.isNull(publisher)) { + try { + publisher = Publisher.newBuilder( ProjectTopicName.newBuilder() - .setProject(this.tenant.getProjectId()) - .setTopic(PUBSUB_SEARCH_TOPIC).build()) + .setProject(this.tenant.getProjectId()) + .setTopic(PUBSUB_SEARCH_TOPIC).build()) .setRetrySettings(RETRY_SETTINGS).build(); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error has occurred.", e); + } } final int BATCH_SIZE = 50; -- GitLab From 10d400532b10c3bb88b37e9d378ced08d24a2107 Mon Sep 17 00:00:00 2001 From: Anastasiia_Gelmut Date: Mon, 14 Dec 2020 11:54:06 +0400 Subject: [PATCH 15/44] GONRG-1365 Fixed Sonar Comments --- .../provider/gcp/GoogleCloudStorage.java | 59 ++++++++++--------- .../storage/provider/gcp/GooglePubSub.java | 33 ++++++----- .../provider/gcp/SomeBasicInterfaceImpl.java | 28 --------- .../gcp/middleware/GcpExceptionMapper.java | 17 ++++++ .../gcp/util/ServiceAccountJwtClientImpl.java | 37 ++++++------ 5 files changed, 87 insertions(+), 87 deletions(-) delete mode 100644 provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/SomeBasicInterfaceImpl.java diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java index bb156101..50313d1a 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java @@ -1,16 +1,19 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + Copyright 2020 Google LLC + Copyright 2020 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ package org.opengroup.osdu.storage.provider.gcp; @@ -53,15 +56,17 @@ import static org.apache.commons.codec.binary.Base64.encodeBase64; public class GoogleCloudStorage implements ICloudStorage { private static final String RECORD_WRITING_ERROR_REASON = "Error on writing record"; + private static final String RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG = "Record %s does not have versions available"; + private static final String ERROR_ON_WRITING_THE_RECORD_HAS_OCCURRED_MSG = "An unexpected error on writing the record has occurred"; @Value("${PUBSUB_SEARCH_TOPIC}") - public String PUBSUB_SEARCH_TOPIC; + public String pubsubSearchTopic; @Value("${GOOGLE_AUDIENCES}") - public String GOOGLE_AUDIENCES; + public String googleAudiences; @Value("${STORAGE_HOSTNAME}") - public String STORAGE_HOSTNAME; + public String storageHostname; @Autowired private DpsHeaders headers; @@ -83,7 +88,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public void write(RecordProcessing... records) { - String bucket = this.getBucketName(this.tenant); + String bucket = getBucketName(this.tenant); ObjectMapper mapper = new ObjectMapper(); @@ -113,7 +118,7 @@ public class GoogleCloudStorage implements ICloudStorage { throw (AppException) e.getCause(); } else { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion", - "An unexpected error on writing the record has occurred", e); + ERROR_ON_WRITING_THE_RECORD_HAS_OCCURRED_MSG, e); } } } @@ -166,14 +171,14 @@ public class GoogleCloudStorage implements ICloudStorage { return true; } - String bucket = this.getBucketName(this.tenant); + String bucket = getBucketName(this.tenant); for (RecordMetadata record : records) { if (!record.getStatus().equals(RecordState.active)) { continue; } if (!record.hasVersion()) { - this.log.warning(String.format("Record %s does not have versions available", record.getId())); + this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId())); continue; } @@ -225,7 +230,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String path = record.getVersionPath(version); - byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()).readAllBytes(this.getBucketName(this.tenant), path); + byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()).readAllBytes(getBucketName(this.tenant), path); return new String(blob, UTF_8); } catch (StorageException e) { @@ -245,7 +250,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public Map read(Map objects) { - String bucketName = this.getBucketName(this.tenant); + String bucketName = getBucketName(this.tenant); Map map = new HashMap<>(); @@ -273,7 +278,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public Map getHash(Collection records) { - String bucket = this.getBucketName(this.tenant); + String bucket = getBucketName(this.tenant); BlobId[] blobIds = records .stream() @@ -298,12 +303,12 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public void delete(RecordMetadata record) { if (!record.hasVersion()) { - this.log.warning(String.format("Record %s does not have versions available", record.getId())); + this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId())); return; } boolean mustSubmit = false; - String bucket = this.getBucketName(this.tenant); + String bucket = getBucketName(this.tenant); Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()); StorageBatch batch = storage.batch(); @@ -336,14 +341,14 @@ public class GoogleCloudStorage implements ICloudStorage { public void deleteVersion(RecordMetadata record, Long version) { boolean mustSubmit = false; - String bucket = this.getBucketName(this.tenant); + String bucket = getBucketName(this.tenant); Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), this.tenant.getServiceAccount(), this.tenant.getProjectId(), this.tenant.getName()); StorageBatch batch = storage.batch(); try { if (!record.hasVersion()) { - this.log.warning(String.format("Record %s does not have versions available", record.getId())); + this.log.warning(String.format(RECORD_DOES_NOT_HAVE_VERSIONS_AVAILABLE_MSG, record.getId())); } Blob blob = storage.get(BlobId.of(bucket, record.getVersionPath(version))); diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java index 994a9fdb..e0cd06f3 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GooglePubSub.java @@ -1,16 +1,19 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + Copyright 2020 Google LLC + Copyright 2020 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ package org.opengroup.osdu.storage.provider.gcp; @@ -38,7 +41,7 @@ import java.util.Arrays; public class GooglePubSub implements IMessageBus { @Value("${PUBSUB_SEARCH_TOPIC}") - public String PUBSUB_SEARCH_TOPIC; + public String pubsubSearchTopic; private Publisher publisher; @@ -64,7 +67,7 @@ public class GooglePubSub implements IMessageBus { publisher = Publisher.newBuilder( ProjectTopicName.newBuilder() .setProject(this.tenant.getProjectId()) - .setTopic(PUBSUB_SEARCH_TOPIC).build()) + .setTopic(pubsubSearchTopic).build()) .setRetrySettings(RETRY_SETTINGS).build(); } catch (Exception e) { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/SomeBasicInterfaceImpl.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/SomeBasicInterfaceImpl.java deleted file mode 100644 index 0609c985..00000000 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/SomeBasicInterfaceImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.storage.provider.gcp; - -import org.opengroup.osdu.storage.provider.interfaces.ISomeBasicInterface; -import org.springframework.stereotype.Component; - -@Component -public class SomeBasicInterfaceImpl implements ISomeBasicInterface { - @Override - public String hello() { - return "eeeee"; - } - - private String thisIsGcpSubmodule = "gcp"; -} diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/middleware/GcpExceptionMapper.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/middleware/GcpExceptionMapper.java index 81b2cdd2..7496839b 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/middleware/GcpExceptionMapper.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/middleware/GcpExceptionMapper.java @@ -1,3 +1,20 @@ +/* + Copyright 2020 Google LLC + Copyright 2020 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + package org.opengroup.osdu.storage.provider.gcp.middleware; import com.google.cloud.storage.StorageException; diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/util/ServiceAccountJwtClientImpl.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/util/ServiceAccountJwtClientImpl.java index 7a28424f..ac2c7cdc 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/util/ServiceAccountJwtClientImpl.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/util/ServiceAccountJwtClientImpl.java @@ -1,16 +1,19 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + Copyright 2020 Google LLC + Copyright 2020 EPAM Systems, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ package org.opengroup.osdu.storage.provider.gcp.util; @@ -73,10 +76,10 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { private JaxRsDpsLog logger; @Value("${STORAGE_HOSTNAME}") - public String STORAGE_HOSTNAME; + public String storageHostname; @Value("${GOOGLE_AUDIENCES}") - public String GOOGLE_AUDIENCES; + public String googleAudiences; @Override public String getIdToken(String tenantName) { @@ -147,14 +150,14 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { } this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, new HttpCredentialsAdapter(credential)) - .setApplicationName(STORAGE_HOSTNAME).build(); + .setApplicationName(storageHostname).build(); } return this.iam; } private Map getJwtCreationPayload(TenantInfo tenantInfo) { - String googleAudience = GOOGLE_AUDIENCES; + String googleAudience = googleAudiences; if (googleAudience.contains(",")) { googleAudience = googleAudience.split(",")[0]; } -- GitLab From facd5b0310140c670b518c4925e49f6b99d813b9 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Mon, 21 Dec 2020 13:00:24 +0400 Subject: [PATCH 16/44] GONRG-1419: Add configuration option to access GCS as service-account and not as a user https://jiraeu.epam.com/browse/GONRG-1419 * Added configuration for impersonalized mode * Updated `core-lib-gcp` to the latest version --- provider/storage-gcp/pom.xml | 2 +- .../provider/gcp/GoogleCloudStorage.java | 25 +++++++++++-------- .../main/resources/application-dev.properties | 2 ++ .../resources/application-kuber.properties | 2 ++ .../resources/application-local.properties | 1 + .../resources/application-osdu-gcp.properties | 2 ++ .../resources/application-testing.properties | 2 ++ .../src/main/resources/application.properties | 2 ++ .../provider/gcp/GoogleCloudStorageTest.java | 2 +- 9 files changed, 27 insertions(+), 13 deletions(-) diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 007680d2..497b14d2 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -42,7 +42,7 @@ org.opengroup.osdu core-lib-gcp - 0.3.24 + 0.3.25 diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java index 50313d1a..263ef864 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java @@ -68,6 +68,9 @@ public class GoogleCloudStorage implements ICloudStorage { @Value("${STORAGE_HOSTNAME}") public String storageHostname; + @Value("#{new Boolean('${osdu.gcp.storage.gcs.enable-impersonalization:false}')}") + private Boolean isImpersonalized; + @Autowired private DpsHeaders headers; @@ -126,7 +129,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public Map updateObjectMetadata(List recordsMetadata, List recordsId, List validMetadata, List lockedRecords, Map recordsIdMap) { String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); Map originalAcls = new HashMap<>(); Map currentRecords = this.recordRepository.get(recordsId); @@ -155,7 +158,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public void revertObjectMetadata(List recordsMetadata, Map originalAcls) { String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); for (RecordMetadata recordMetadata : recordsMetadata) { Blob blob = storage.get(bucket, recordMetadata.getVersionPath(recordMetadata.getLatestVersion())); @@ -184,7 +187,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String path = record.getVersionPath(record.getLatestVersion()); - Blob blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()).get(bucket, path); + Blob blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).get(bucket, path); if (blob == null) { throw new StorageException(HttpStatus.SC_NOT_FOUND, String.format("'%s' not found", path)); } @@ -193,7 +196,7 @@ public class GoogleCloudStorage implements ICloudStorage { // inconsistency then cleanup and check the access again // This makes all the APIs robust to inconsistent data, but will add some // latency - if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()))) { + if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized))) { return false; } } @@ -210,7 +213,7 @@ public class GoogleCloudStorage implements ICloudStorage { // get meaning data has been corrupted // (This is ok for DR, because DR service only revoke the data store written // permission from datafier) - Storage storageClientDatafierCredential = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()); + Storage storageClientDatafierCredential = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); if (storageClientDatafierCredential .get(BlobId.of(bucket, record.getVersionPath(record.getLatestVersion()))) != null) { return false; @@ -230,7 +233,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String path = record.getVersionPath(version); - byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()).readAllBytes(getBucketName(this.tenant), path); + byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).readAllBytes(getBucketName(this.tenant), path); return new String(blob, UTF_8); } catch (StorageException e) { @@ -285,7 +288,7 @@ public class GoogleCloudStorage implements ICloudStorage { .map(rm -> BlobId.of(bucket, rm.getVersionPath(rm.getLatestVersion()))) .toArray(BlobId[]::new); - List blobs = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()).get(blobIds); + List blobs = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).get(blobIds); Map hashes = new HashMap<>(); @@ -309,7 +312,7 @@ public class GoogleCloudStorage implements ICloudStorage { boolean mustSubmit = false; String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName()); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); StorageBatch batch = storage.batch(); @@ -342,7 +345,7 @@ public class GoogleCloudStorage implements ICloudStorage { boolean mustSubmit = false; String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), this.tenant.getServiceAccount(), this.tenant.getProjectId(), this.tenant.getName()); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), this.tenant.getServiceAccount(), this.tenant.getProjectId(), this.tenant.getName(), isImpersonalized); StorageBatch batch = storage.batch(); @@ -408,7 +411,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String content = mapper.writeValueAsString(processing.getRecordData()); - this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName).create(blobInfo, content.getBytes(UTF_8)); + this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, isImpersonalized).create(blobInfo, content.getBytes(UTF_8)); } catch (StorageException e) { if (e.getCode() == HttpStatus.SC_BAD_REQUEST) { throw new AppException(HttpStatus.SC_BAD_REQUEST, RECORD_WRITING_ERROR_REASON, e.getMessage(), e); @@ -434,7 +437,7 @@ public class GoogleCloudStorage implements ICloudStorage { String key = tokens[tokens.length - 2]; try { - String value = new String(this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName).readAllBytes(bucket, object), UTF_8); + String value = new String(this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, isImpersonalized).readAllBytes(bucket, object), UTF_8); map.put(key, value); } catch (StorageException e) { map.put(key, null); diff --git a/provider/storage-gcp/src/main/resources/application-dev.properties b/provider/storage-gcp/src/main/resources/application-dev.properties index 9f938ab0..cb7a8330 100644 --- a/provider/storage-gcp/src/main/resources/application-dev.properties +++ b/provider/storage-gcp/src/main/resources/application-dev.properties @@ -8,4 +8,6 @@ AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes.appspot.com/api/crs/v2 +osdu.gcp.storage.gcs.enable-impersonalization=false + DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-kuber.properties b/provider/storage-gcp/src/main/resources/application-kuber.properties index 68fc03ae..155d98a5 100644 --- a/provider/storage-gcp/src/main/resources/application-kuber.properties +++ b/provider/storage-gcp/src/main/resources/application-kuber.properties @@ -10,4 +10,6 @@ CRS_API=${CRS_API} GOOGLE_AUDIENCES=${GOOGLE_AUDIENCES} DEFAULT_DATA_COUNTRY=US +osdu.gcp.storage.gcs.enable-impersonalization=false + disable.appengine.log.factory=true \ No newline at end of file diff --git a/provider/storage-gcp/src/main/resources/application-local.properties b/provider/storage-gcp/src/main/resources/application-local.properties index f4b78aa2..f25b0500 100644 --- a/provider/storage-gcp/src/main/resources/application-local.properties +++ b/provider/storage-gcp/src/main/resources/application-local.properties @@ -8,4 +8,5 @@ AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes.appspot.com/api/crs/v2 +osdu.gcp.storage.gcs.enable-impersonalization=false DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties b/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties index 1af31a0b..c2b5905c 100644 --- a/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties +++ b/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties @@ -9,4 +9,6 @@ AUTHORIZE_API=https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-nice-etching-277309.uc.r.appspot.com/api/crs/v2 +osdu.gcp.storage.gcs.enable-impersonalization=false + DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-testing.properties b/provider/storage-gcp/src/main/resources/application-testing.properties index c2fb8405..84e4c4b4 100644 --- a/provider/storage-gcp/src/main/resources/application-testing.properties +++ b/provider/storage-gcp/src/main/resources/application-testing.properties @@ -9,4 +9,6 @@ AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v2 +osdu.gcp.storage.gcs.enable-impersonalization=false + DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application.properties b/provider/storage-gcp/src/main/resources/application.properties index d1f2efdc..57dda0d9 100644 --- a/provider/storage-gcp/src/main/resources/application.properties +++ b/provider/storage-gcp/src/main/resources/application.properties @@ -1,5 +1,7 @@ LOG_PREFIX=storage +osdu.gcp.storage.gcs.enable-impersonalization=false + server.servlet.contextPath=/api/storage/v2/ logging.level.org.springframework.web=DEBUG server.port=8080 diff --git a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java index f909ef34..43c996cb 100644 --- a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java +++ b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java @@ -79,7 +79,7 @@ public class GoogleCloudStorageTest { @SuppressWarnings("unchecked") public void setup() throws Exception { - when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any())).thenReturn(this.storage); + when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any(), anyBoolean())).thenReturn(this.storage); this.acl = new Acl(); this.acl.setViewers(new String[]{ACL_VIEWER_1, ACL_VIEWER_2}); -- GitLab From f00d926d1d3cf67b60a8e867537a3f7e179da1ae Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 22 Dec 2020 09:53:10 +0400 Subject: [PATCH 17/44] GONRG-1419: Add configuration option to access GCS as service-account and not as a user https://jiraeu.epam.com/browse/GONRG-1419 * Added configuration for impersonalized mode * Updated `core-lib-gcp` to the latest version --- .../storage-gcp/src/main/resources/application-dev.properties | 2 -- .../storage-gcp/src/main/resources/application-kuber.properties | 2 -- .../storage-gcp/src/main/resources/application-local.properties | 1 - .../src/main/resources/application-osdu-gcp.properties | 2 -- .../src/main/resources/application-testing.properties | 2 -- 5 files changed, 9 deletions(-) diff --git a/provider/storage-gcp/src/main/resources/application-dev.properties b/provider/storage-gcp/src/main/resources/application-dev.properties index cb7a8330..9f938ab0 100644 --- a/provider/storage-gcp/src/main/resources/application-dev.properties +++ b/provider/storage-gcp/src/main/resources/application-dev.properties @@ -8,6 +8,4 @@ AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes.appspot.com/api/crs/v2 -osdu.gcp.storage.gcs.enable-impersonalization=false - DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-kuber.properties b/provider/storage-gcp/src/main/resources/application-kuber.properties index 155d98a5..68fc03ae 100644 --- a/provider/storage-gcp/src/main/resources/application-kuber.properties +++ b/provider/storage-gcp/src/main/resources/application-kuber.properties @@ -10,6 +10,4 @@ CRS_API=${CRS_API} GOOGLE_AUDIENCES=${GOOGLE_AUDIENCES} DEFAULT_DATA_COUNTRY=US -osdu.gcp.storage.gcs.enable-impersonalization=false - disable.appengine.log.factory=true \ No newline at end of file diff --git a/provider/storage-gcp/src/main/resources/application-local.properties b/provider/storage-gcp/src/main/resources/application-local.properties index f25b0500..f4b78aa2 100644 --- a/provider/storage-gcp/src/main/resources/application-local.properties +++ b/provider/storage-gcp/src/main/resources/application-local.properties @@ -8,5 +8,4 @@ AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes.appspot.com/api/crs/v2 -osdu.gcp.storage.gcs.enable-impersonalization=false DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties b/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties index c2b5905c..1af31a0b 100644 --- a/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties +++ b/provider/storage-gcp/src/main/resources/application-osdu-gcp.properties @@ -9,6 +9,4 @@ AUTHORIZE_API=https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-nice-etching-277309.uc.r.appspot.com/api/crs/v2 -osdu.gcp.storage.gcs.enable-impersonalization=false - DEFAULT_DATA_COUNTRY=US diff --git a/provider/storage-gcp/src/main/resources/application-testing.properties b/provider/storage-gcp/src/main/resources/application-testing.properties index 84e4c4b4..c2fb8405 100644 --- a/provider/storage-gcp/src/main/resources/application-testing.properties +++ b/provider/storage-gcp/src/main/resources/application-testing.properties @@ -9,6 +9,4 @@ AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1 LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1 CRS_API=https://os-crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v2 -osdu.gcp.storage.gcs.enable-impersonalization=false - DEFAULT_DATA_COUNTRY=US -- GitLab From bc0c5b72e4ea1d53e154bb4c7b491f539b295523 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 22 Dec 2020 17:44:08 +0400 Subject: [PATCH 18/44] GONRG-1419: Add configuration option to access GCS as service-account and not as a user https://jiraeu.epam.com/browse/GONRG-1419 * Added configuration for impersonalized mode * Updated `core-lib-gcp` to the latest version --- .../osdu/storage/provider/gcp/GoogleCloudStorageTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java index 43c996cb..779e6d16 100644 --- a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java +++ b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java @@ -79,7 +79,7 @@ public class GoogleCloudStorageTest { @SuppressWarnings("unchecked") public void setup() throws Exception { - when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any(), anyBoolean())).thenReturn(this.storage); + when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any(), any())).thenReturn(this.storage); this.acl = new Acl(); this.acl.setViewers(new String[]{ACL_VIEWER_1, ACL_VIEWER_2}); -- GitLab From 3055c4f2e52516115485c2df3889ee7992ef2f53 Mon Sep 17 00:00:00 2001 From: Aliaksandr Ramanovich1 Date: Wed, 23 Dec 2020 06:03:59 +0000 Subject: [PATCH 19/44] Parametrize cloudbuild.yaml --- provider/storage-gcp/cloudbuild/cloudbuild.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/provider/storage-gcp/cloudbuild/cloudbuild.yaml b/provider/storage-gcp/cloudbuild/cloudbuild.yaml index 93149e71..403520f8 100644 --- a/provider/storage-gcp/cloudbuild/cloudbuild.yaml +++ b/provider/storage-gcp/cloudbuild/cloudbuild.yaml @@ -1,4 +1,3 @@ - # Copyright 2020 Google LLC # Copyright 2017-2019, Schlumberger # Copyright 2020 EPAM @@ -21,11 +20,11 @@ steps: 'build', '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}', '--build-arg', 'PORT=${_PORT}', - '-t', 'gcr.io/$PROJECT_ID/os-storage/storage-${_PROVIDER_NAME}:${_SHORT_SHA}', - '-t', 'gcr.io/$PROJECT_ID/os-storage/storage-${_PROVIDER_NAME}:latest', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/storage-${_PROVIDER_NAME}:${_SHORT_SHA}', + '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/storage-${_PROVIDER_NAME}:latest', '-f', 'provider/storage-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild', '.' ] images: - - 'gcr.io/$PROJECT_ID/os-storage/storage-${_PROVIDER_NAME}' + - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/storage-${_PROVIDER_NAME}' -- GitLab From f3b9622fd1e4c1e1ef2c05c8ec730e334aec4f1b Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 29 Dec 2020 01:04:01 +0400 Subject: [PATCH 20/44] GONRG-1419: Add configuration option to access GCS as service-account and not as a user https://jiraeu.epam.com/browse/GONRG-1419 * Added configuration for impersonalized mode * Updated `core-lib-gcp` to the latest version --- .../provider/gcp/GoogleCloudStorage.java | 27 ++++++++-------- .../gcp/config/StorageConfigProperties.java | 32 +++++++++++++++++++ .../provider/gcp/GoogleCloudStorageTest.java | 6 +++- 3 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/config/StorageConfigProperties.java diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java index 263ef864..396d96dc 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java @@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.*; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.gcp.multitenancy.IStorageFactory; import org.opengroup.osdu.core.common.util.Crc32c; +import org.opengroup.osdu.storage.provider.gcp.config.StorageConfigProperties; import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; import org.apache.commons.lang3.ArrayUtils; import org.apache.http.HttpStatus; @@ -68,8 +69,8 @@ public class GoogleCloudStorage implements ICloudStorage { @Value("${STORAGE_HOSTNAME}") public String storageHostname; - @Value("#{new Boolean('${osdu.gcp.storage.gcs.enable-impersonalization:false}')}") - private Boolean isImpersonalized; + @Autowired + private StorageConfigProperties properties; @Autowired private DpsHeaders headers; @@ -129,7 +130,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public Map updateObjectMetadata(List recordsMetadata, List recordsId, List validMetadata, List lockedRecords, Map recordsIdMap) { String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()); Map originalAcls = new HashMap<>(); Map currentRecords = this.recordRepository.get(recordsId); @@ -158,7 +159,7 @@ public class GoogleCloudStorage implements ICloudStorage { @Override public void revertObjectMetadata(List recordsMetadata, Map originalAcls) { String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()); for (RecordMetadata recordMetadata : recordsMetadata) { Blob blob = storage.get(bucket, recordMetadata.getVersionPath(recordMetadata.getLatestVersion())); @@ -187,7 +188,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String path = record.getVersionPath(record.getLatestVersion()); - Blob blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).get(bucket, path); + Blob blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()).get(bucket, path); if (blob == null) { throw new StorageException(HttpStatus.SC_NOT_FOUND, String.format("'%s' not found", path)); } @@ -196,7 +197,7 @@ public class GoogleCloudStorage implements ICloudStorage { // inconsistency then cleanup and check the access again // This makes all the APIs robust to inconsistent data, but will add some // latency - if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized))) { + if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()))) { return false; } } @@ -213,7 +214,7 @@ public class GoogleCloudStorage implements ICloudStorage { // get meaning data has been corrupted // (This is ok for DR, because DR service only revoke the data store written // permission from datafier) - Storage storageClientDatafierCredential = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); + Storage storageClientDatafierCredential = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()); if (storageClientDatafierCredential .get(BlobId.of(bucket, record.getVersionPath(record.getLatestVersion()))) != null) { return false; @@ -233,7 +234,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String path = record.getVersionPath(version); - byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).readAllBytes(getBucketName(this.tenant), path); + byte[] blob = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()).readAllBytes(getBucketName(this.tenant), path); return new String(blob, UTF_8); } catch (StorageException e) { @@ -288,7 +289,7 @@ public class GoogleCloudStorage implements ICloudStorage { .map(rm -> BlobId.of(bucket, rm.getVersionPath(rm.getLatestVersion()))) .toArray(BlobId[]::new); - List blobs = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized).get(blobIds); + List blobs = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()).get(blobIds); Map hashes = new HashMap<>(); @@ -312,7 +313,7 @@ public class GoogleCloudStorage implements ICloudStorage { boolean mustSubmit = false; String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), isImpersonalized); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()); StorageBatch batch = storage.batch(); @@ -345,7 +346,7 @@ public class GoogleCloudStorage implements ICloudStorage { boolean mustSubmit = false; String bucket = getBucketName(this.tenant); - Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), this.tenant.getServiceAccount(), this.tenant.getProjectId(), this.tenant.getName(), isImpersonalized); + Storage storage = this.storageFactory.getStorage(this.headers.getUserEmail(), this.tenant.getServiceAccount(), this.tenant.getProjectId(), this.tenant.getName(), properties.isEnableImpersonalization()); StorageBatch batch = storage.batch(); @@ -411,7 +412,7 @@ public class GoogleCloudStorage implements ICloudStorage { try { String content = mapper.writeValueAsString(processing.getRecordData()); - this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, isImpersonalized).create(blobInfo, content.getBytes(UTF_8)); + this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, properties.isEnableImpersonalization()).create(blobInfo, content.getBytes(UTF_8)); } catch (StorageException e) { if (e.getCode() == HttpStatus.SC_BAD_REQUEST) { throw new AppException(HttpStatus.SC_BAD_REQUEST, RECORD_WRITING_ERROR_REASON, e.getMessage(), e); @@ -437,7 +438,7 @@ public class GoogleCloudStorage implements ICloudStorage { String key = tokens[tokens.length - 2]; try { - String value = new String(this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, isImpersonalized).readAllBytes(bucket, object), UTF_8); + String value = new String(this.storageFactory.getStorage(userId, serviceAccount, projectId, tenantName, properties.isEnableImpersonalization()).readAllBytes(bucket, object), UTF_8); map.put(key, value); } catch (StorageException e) { map.put(key, null); diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/config/StorageConfigProperties.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/config/StorageConfigProperties.java new file mode 100644 index 00000000..f12a702d --- /dev/null +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/config/StorageConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2020 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.gcp.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "osdu.gcp.storage.gcs") +@Getter +@Setter +public class StorageConfigProperties { + + private boolean enableImpersonalization; +} \ No newline at end of file diff --git a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java index 779e6d16..3cb7fe6e 100644 --- a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java +++ b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java @@ -19,6 +19,7 @@ import org.opengroup.osdu.core.common.model.storage.RecordMetadata; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.gcp.multitenancy.IStorageFactory; +import org.opengroup.osdu.storage.provider.gcp.config.StorageConfigProperties; import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; import java.util.*; @@ -57,6 +58,9 @@ public class GoogleCloudStorageTest { @Mock private Storage storage; + @Mock + private StorageConfigProperties properties; + @Mock private IRecordsMetadataRepository recordRepository; @@ -78,7 +82,7 @@ public class GoogleCloudStorageTest { @Before @SuppressWarnings("unchecked") public void setup() throws Exception { - + when(this.properties.isEnableImpersonalization()).thenReturn(false); when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any(), any())).thenReturn(this.storage); this.acl = new Acl(); -- GitLab From 1f19a288e2986fa45d069aa6744ea73e0647d521 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 12 Jan 2021 11:48:41 +0400 Subject: [PATCH 21/44] GONRG-1476: Publish changes from Storage https://jiraeu.epam.com/browse/GONRG-1476 * Changed field-injection to setter-injection --- .../osdu/storage/provider/gcp/GoogleCloudStorage.java | 5 +++++ .../osdu/storage/provider/gcp/GoogleCloudStorageTest.java | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java index 396d96dc..7991c2a2 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java @@ -90,6 +90,11 @@ public class GoogleCloudStorage implements ICloudStorage { @Autowired private JaxRsDpsLog log; + @Autowired + public void setProperties(StorageConfigProperties properties){ + this.properties = properties; + } + @Override public void write(RecordProcessing... records) { String bucket = getBucketName(this.tenant); diff --git a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java index 3cb7fe6e..9b6dcce3 100644 --- a/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java +++ b/provider/storage-gcp/src/test/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorageTest.java @@ -58,9 +58,6 @@ public class GoogleCloudStorageTest { @Mock private Storage storage; - @Mock - private StorageConfigProperties properties; - @Mock private IRecordsMetadataRepository recordRepository; @@ -82,7 +79,10 @@ public class GoogleCloudStorageTest { @Before @SuppressWarnings("unchecked") public void setup() throws Exception { - when(this.properties.isEnableImpersonalization()).thenReturn(false); + StorageConfigProperties properties = new StorageConfigProperties(); + properties.setEnableImpersonalization(false); + sut.setProperties(properties); + when(this.storageFactory.getStorage(any(), eq(SERVICE_ACCOUNT), eq(PROJECT_ID), any(), any())).thenReturn(this.storage); this.acl = new Acl(); -- GitLab From 6435fa8005fba2ab08a84e9f4e521ec02980dc02 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Fri, 15 Jan 2021 13:11:03 +0400 Subject: [PATCH 22/44] Added config properties. Updated Copyright. Refactoring code. --- .../provider/reference/CloudStorageImpl.java | 381 ++++++++++-------- .../app/StorageReferenceApplication.java | 19 +- .../provider/reference/cache/GroupCache.java | 32 +- .../reference/cache/LegalTagCache.java | 17 + .../provider/reference/cache/SchemaCache.java | 17 + .../config/MinIoConfigProperties.java | 38 ++ .../config/MongoDBConfigProperties.java | 35 ++ .../config/RabbitMqConfigProperties.java | 32 ++ .../config/RedisConfigProperties.java | 34 ++ .../reference/di/RabbitMQFactoryImpl.java | 39 +- .../reference/di/TenantFactoryImpl.java | 20 +- .../factory/CloudObjectStorageFactory.java | 56 ++- .../LegalComplianceChangeServiceImpl.java | 25 +- .../reference/messagebus/IMessageFactory.java | 18 +- .../model/RecordMetadataDocument.java | 17 + .../reference/model/SchemaDocument.java | 17 + .../reference/model/TenantInfoDocument.java | 17 + .../persistence/MongoDdmsClient.java | 25 +- .../repository/QueryRepositoryImpl.java | 31 +- .../RecordsMetadataRepositoryImpl.java | 27 +- .../repository/SchemaRepositoryImpl.java | 23 +- .../security/BasicAuthSecurityConfig.java | 17 + .../reference/security/WhoamiController.java | 17 + .../service/BatchServiceReferenceImpl.java | 31 +- .../reference/service/MessageBusImpl.java | 25 +- .../reference/util/MongoClientHandler.java | 44 +- .../util/ServiceAccountJwtClientImpl.java | 26 +- .../src/main/resources/application.properties | 26 +- .../reference/CloudStorageImplTest.java | 57 ++- .../context/ProviderSpringContextTest.java | 20 +- 30 files changed, 892 insertions(+), 291 deletions(-) create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java index 3c2f1db9..210dfed5 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java @@ -1,5 +1,24 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference; +import static org.apache.commons.codec.binary.Base64.encodeBase64; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.minio.GetObjectArgs; @@ -10,12 +29,13 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.PostConstruct; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.storage.RecordData; import org.opengroup.osdu.core.common.model.storage.RecordMetadata; @@ -24,195 +44,204 @@ import org.opengroup.osdu.core.common.model.storage.RecordState; import org.opengroup.osdu.core.common.model.storage.TransferInfo; import org.opengroup.osdu.core.common.util.Crc32c; import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; -import org.apache.http.HttpStatus; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.stereotype.Repository; -import java.nio.charset.StandardCharsets; - -import static org.apache.commons.codec.binary.Base64.encodeBase64; - @Repository public class CloudStorageImpl implements ICloudStorage { - @Value("${minio.bucket.record.name}") - private String recordBucketName; - - @Autowired - private CloudObjectStorageFactory factory; - - @Autowired - private JaxRsDpsLog log; - - private MinioClient minioClient; - - @PostConstruct - public void init() { - minioClient = factory.getClient(); - } - - @Override - public void write(RecordProcessing... recordsProcessing) { - Gson gson = new GsonBuilder().serializeNulls().create(); - for (RecordProcessing rp: recordsProcessing) { - Map headers = new HashMap<>(); - headers.put("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE); - headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); - - String content = gson.toJson(rp.getRecordData()); - byte[] bytes = content.getBytes(StandardCharsets.UTF_8); - String itemName = getItemName(rp.getRecordMetadata()).replace(":", "-"); - try { - minioClient.putObject( - PutObjectArgs.builder().bucket(recordBucketName).object(itemName).stream( - new ByteArrayInputStream(bytes), bytes.length, -1) - .headers(headers) - .build()); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, - "Failed to write new record.", e.getMessage()); - } - } - } - - @Override - public Map getHash(Collection records) { - Gson gson = new Gson(); - Map hashes = new HashMap<>(); - for (RecordMetadata rm : records) { - String jsonData = read(rm, rm.getLatestVersion(), false); - RecordData data = gson.fromJson(jsonData, RecordData.class); - - String hash = getHash(data); - hashes.put(rm.getId(), hash); - } - return hashes; - } - - @Override - public boolean isDuplicateRecord(TransferInfo transfer, Map hashMap, Map.Entry kv) { - RecordMetadata updatedRecordMetadata = kv.getKey(); - RecordData recordData = kv.getValue(); - String recordHash = hashMap.get(updatedRecordMetadata.getId()); - - String newHash = getHash(recordData); - - if (newHash.equals(recordHash)) { - transfer.getSkippedRecords().add(updatedRecordMetadata.getId()); - return true; - }else{ - return false; - } + private static final Logger LOGGER = LoggerFactory.getLogger(CloudStorageImpl.class); + + private final MinIoConfigProperties minIoConfigProperties; + private final CloudObjectStorageFactory factory; + + private MinioClient minioClient; + + public CloudStorageImpl(CloudObjectStorageFactory factory, + MinIoConfigProperties minIoConfigProperties) { + this.factory = factory; + this.minIoConfigProperties = minIoConfigProperties; + } + + @PostConstruct + public void init() { + minioClient = factory.getClient(); + } + + @Override + public void write(RecordProcessing... recordsProcessing) { + Gson gson = new GsonBuilder().serializeNulls().create(); + for (RecordProcessing rp : recordsProcessing) { + Map headers = new HashMap<>(); + headers.put("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE); + headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); + + String content = gson.toJson(rp.getRecordData()); + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + String itemName = getItemName(rp.getRecordMetadata()).replace(":", "-"); + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .stream(new ByteArrayInputStream(bytes), bytes.length, -1) + .headers(headers) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, + "Failed to write new record.", e.getMessage()); + } } - - private String getHash(RecordData data) { - Gson gson = new Gson(); - Crc32c checksumGenerator = new Crc32c(); - - String newRecordStr = gson.toJson(data); - byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8); - checksumGenerator.update(bytes, 0, bytes.length); - bytes = checksumGenerator.getValueAsBytes(); - String newHash = new String(encodeBase64(bytes)); - return newHash; + } + + @Override + public Map getHash(Collection records) { + Gson gson = new Gson(); + Map hashes = new HashMap<>(); + for (RecordMetadata rm : records) { + String jsonData = read(rm, rm.getLatestVersion(), false); + RecordData data = gson.fromJson(jsonData, RecordData.class); + + String hash = getHash(data); + hashes.put(rm.getId(), hash); } - - private String getItemName(RecordMetadata record) { - return record.getVersionPath(record.getLatestVersion()); + return hashes; + } + + @Override + public boolean isDuplicateRecord(TransferInfo transfer, Map hashMap, + Map.Entry kv) { + RecordMetadata updatedRecordMetadata = kv.getKey(); + RecordData recordData = kv.getValue(); + String recordHash = hashMap.get(updatedRecordMetadata.getId()); + + String newHash = getHash(recordData); + + if (newHash.equals(recordHash)) { + transfer.getSkippedRecords().add(updatedRecordMetadata.getId()); + return true; + } else { + return false; } - - private String getItemName(RecordMetadata record, Long version) { - return record.getVersionPath(version); + } + + private String getHash(RecordData data) { + Gson gson = new Gson(); + Crc32c checksumGenerator = new Crc32c(); + + String newRecordStr = gson.toJson(data); + byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8); + checksumGenerator.update(bytes, 0, bytes.length); + bytes = checksumGenerator.getValueAsBytes(); + String newHash = new String(encodeBase64(bytes)); + return newHash; + } + + private String getItemName(RecordMetadata record) { + return record.getVersionPath(record.getLatestVersion()); + } + + private String getItemName(RecordMetadata record, Long version) { + return record.getVersionPath(version); + } + + @Override + public void delete(RecordMetadata record) { + String itemName = getItemName(record).replace(":", "-"); + try { + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete record.", + e.getMessage()); } - - public void setRecordBucketName(String recordBucketName) { - this.recordBucketName = recordBucketName; + } + + @Override + public void deleteVersion(RecordMetadata record, Long version) { + String itemName = getItemName(record, version).replace(":", "-"); + try { + if (!record.hasVersion()) { + LOGGER.warn(String.format("Record %s does not have versions available", record.getId())); + } + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete version.", + e.getMessage()); } - - @Override - public void delete(RecordMetadata record) { - String itemName = getItemName(record).replace(":", "-"); - try { - minioClient.removeObject( - RemoveObjectArgs.builder().bucket(recordBucketName).object(itemName).build()); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete record.", e.getMessage()); - } - } - - @Override - public void deleteVersion(RecordMetadata record, Long version) { - String itemName = getItemName(record, version).replace(":", "-"); - try { - if (!record.hasVersion()) { - log.warning(String.format("Record %s does not have versions available", record.getId())); - } - minioClient.removeObject( - RemoveObjectArgs.builder().bucket(recordBucketName).object(itemName).build()); - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete version.", e.getMessage()); - } - } - - @Override - public boolean hasAccess(RecordMetadata... records) { - for (RecordMetadata record : records) - { - if (!record.getStatus().equals(RecordState.active)) - return false; - } - return true; + } + + @Override + public boolean hasAccess(RecordMetadata... records) { + for (RecordMetadata record : records) { + if (!record.getStatus().equals(RecordState.active)) { + return false; + } } - - @Override - public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) { - String itemName = getItemName(record, version).replace(":", "-"); - String msg = String.format("Record with id '%s' does not exist, version: %s", record.getId(), version); - InputStream stream; - try { - stream = minioClient.getObject( - GetObjectArgs.builder() - .bucket(recordBucketName) - .object(itemName) - .build()); - if (stream == null) { - log.warning(msg); - } else { - return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); - } - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", e.getMessage()); - } - throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg); + return true; + } + + @Override + public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) { + String itemName = getItemName(record, version).replace(":", "-"); + String msg = String + .format("Record with id '%s' does not exist, version: %s", record.getId(), version); + InputStream stream; + try { + stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + if (stream == null) { + LOGGER.warn(msg); + } else { + return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", + e.getMessage()); } - - @Override - public Map read(Map objects) { - // key -> record id - // value -> record version path - Map map = new HashMap<>(); - for (Map.Entry record : objects.entrySet()) { - String[] tokens = record.getValue().split("/"); - String key = tokens[tokens.length - 2]; - try { - InputStream stream = minioClient.getObject( - GetObjectArgs.builder() - .bucket(recordBucketName) - .object(record.getValue().replace(":", "-")) - .build()); - if (stream != null) { - String result = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); - map.put(key, result); - } - } catch (Exception e) { - throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", e.getMessage()); - } + throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg); + } + + @Override + public Map read(Map objects) { + // key -> record id + // value -> record version path + Map map = new HashMap<>(); + for (Map.Entry record : objects.entrySet()) { + String[] tokens = record.getValue().split("/"); + String key = tokens[tokens.length - 2]; + try { + InputStream stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(record.getValue().replace(":", "-")) + .build()); + if (stream != null) { + String result = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + map.put(key, result); } - return map; + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", + e.getMessage()); + } } + return map; + } } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java index 4767155b..590ab40c 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.app; import org.springframework.boot.SpringApplication; @@ -5,7 +22,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import org.springframework.context.annotation.ComponentScan; -@SpringBootApplication(exclude={MongoAutoConfiguration.class}) +@SpringBootApplication(exclude = {MongoAutoConfiguration.class}) @ComponentScan({"org.opengroup.osdu"}) public class StorageReferenceApplication { diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java index 291d512a..60eaab2f 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java @@ -1,17 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.cache; import org.opengroup.osdu.core.common.cache.RedisCache; import org.opengroup.osdu.core.common.model.entitlements.Groups; -import org.springframework.beans.factory.annotation.Value; +import org.opengroup.osdu.storage.provider.reference.config.RedisConfigProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class GroupCache extends RedisCache { - public GroupCache( - @Value("${gcp.redis.host}") final String redisHost, - @Value("${gcp.redis.port}") final Integer redisPort, - @Value("${gcp.redis.exp.time}") final Integer expTimeSec) { - super(redisHost, redisPort, expTimeSec, String.class, Groups.class); - } + @Autowired + public GroupCache(RedisConfigProperties redisConfigProperties) { + super(redisConfigProperties.getGcpRedisHost(), + Integer.parseInt(redisConfigProperties.getGcpRedisPort()), + Integer.parseInt(redisConfigProperties.getGcpRedisExpTime()), String.class, Groups.class); + } } \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java index 685c21b9..0388780b 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.cache; import org.opengroup.osdu.core.common.cache.ICache; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java index deda3ff2..e95694fd 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.cache; import org.opengroup.osdu.core.common.cache.VmCache; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java new file mode 100644 index 00000000..d5000b77 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MinIoConfigProperties { + + private String minIoSignedUrlExpirationDays; + private String minIoEndpointUrl; + private String minIoAccessKey; + private String minIoSecretKey; + private String minIoRegion; + private String minIoPrefix; + private String minIoBucketRecordName; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java new file mode 100644 index 00000000..5438065e --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MongoDBConfigProperties { + + private String mongoDbUrl; + private String mongoDbUser; + private String mongoDbPassword; + private String mongoDbName; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java new file mode 100644 index 00000000..f72efd6b --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RabbitMqConfigProperties { + + private String mbRabbitMqUri; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java new file mode 100644 index 00000000..76455e49 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RedisConfigProperties { + + private String gcpRedisPort; + private String gcpRedisHost; + private String gcpRedisExpTime; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java index 69a1e4b3..0e547678 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.di; import com.rabbitmq.client.Channel; @@ -10,10 +27,11 @@ import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.concurrent.TimeoutException; import javax.annotation.PostConstruct; +import org.opengroup.osdu.storage.provider.reference.config.RabbitMqConfigProperties; import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -22,24 +40,29 @@ import org.springframework.stereotype.Component; public class RabbitMQFactoryImpl implements IMessageFactory { private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); - - @Value("${mb.rabbitmq.uri}") - private String uri; + private final RabbitMqConfigProperties rabbitMqConfigProperties; private Channel channel; + @Autowired + public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) { + this.rabbitMqConfigProperties = rabbitMqConfigProperties; + } + @PostConstruct private void init() { ConnectionFactory factory = new ConnectionFactory(); try { - factory.setUri(uri); + String mbRabbitMqUri = rabbitMqConfigProperties.getMbRabbitMqUri(); + LOG.debug(String.format("RabbitMQ Uri = %s", mbRabbitMqUri)); + factory.setUri(mbRabbitMqUri); factory.setAutomaticRecoveryEnabled(true); Connection conn = factory.newConnection(); this.channel = conn.createChannel(); LOG.debug("RabbitMQ Channel was created."); for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { channel.queueDeclare(queue, true, false, false, null); - LOG.debug("Queue [" + queue + "] was declared."); + LOG.debug(String.format("Queue [ %s ] was declared.", queue)); } } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { LOG.error(e.getMessage(), e); @@ -56,9 +79,9 @@ public class RabbitMQFactoryImpl implements IMessageFactory { String queueNameWithPrefix = queueName; try { channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); - LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]"); + LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix)); } catch (IOException e) { - LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]"); + LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix)); LOG.error(e.getMessage(), e); } } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java index 6473d132..4307d1ca 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.di; import static java.util.Objects.isNull; @@ -8,7 +25,6 @@ import com.mongodb.client.MongoCollection; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.bson.Document; import org.bson.types.ObjectId; @@ -71,7 +87,7 @@ public class TenantFactoryImpl implements ITenantFactory { LOG.error(String.format("Collection \'%s\' is empty.", results)); } for (Document document : results) { - TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class); + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class); ObjectId id = (ObjectId) document.get("_id"); tenantInfo.setId((long) id.getCounter()); tenantInfo.setCrmAccountIds((ArrayList) document.get("crmAccountID")); diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java index 5a4dbcf8..c4db1f81 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java @@ -1,45 +1,61 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.factory; import io.minio.MinioClient; import javax.annotation.PostConstruct; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @Component @Lazy public class CloudObjectStorageFactory { - private static final Logger logger = LoggerFactory.getLogger(CloudObjectStorageFactory.class); - - @Value("${minio.endpoint.url}") - private String endpointURL; - @Value("${minio.access.key}") - private String accessKey; - @Value("${minio.secret.key}") - private String secretKey; - @Value("${minio.region:us-east-1}") - private String region; - @Value("${minio.prefix:local-dev}") - private String bucketNamePrefix; - private MinioClient minioClient; + private static final Logger LOGGER = LoggerFactory.getLogger(CloudObjectStorageFactory.class); + private final MinIoConfigProperties minIoConfigProperties; - private String bucketName; + private MinioClient minioClient; - public CloudObjectStorageFactory() { } + @Autowired + public CloudObjectStorageFactory(MinIoConfigProperties minIoConfigProperties) { + this.minIoConfigProperties = minIoConfigProperties; + } @PostConstruct public void init() { minioClient = MinioClient.builder() - .endpoint(endpointURL) - .credentials(accessKey, secretKey) - .region(region).build(); - logger.info("Minio client initialized"); + .endpoint(minIoConfigProperties.getMinIoEndpointUrl()) + .credentials(minIoConfigProperties.getMinIoAccessKey(), + minIoConfigProperties.getMinIoSecretKey()) + .region(minIoConfigProperties.getMinIoRegion()) + .build(); + LOGGER.info("Minio client initialized"); } public MinioClient getClient() { return this.minioClient; } + + public void setMinioClient(MinioClient minioClient) { + this.minioClient = minioClient; + } } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java index 61ec94b1..fe78a970 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.jobs; import static java.util.Collections.singletonList; @@ -34,7 +51,7 @@ public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeS private IRecordsMetadataRepository recordsRepo; @Autowired - private IMessageBus pubSubclient; + private IMessageBus pubSubClient; @Autowired private StorageAuditLogger auditLogger; @@ -64,7 +81,7 @@ public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeS if (results.getValue() != null && !results.getValue().isEmpty()) { List recordsMetadata = results.getValue(); - PubSubInfo[] pubsubInfos = this + PubSubInfo[] pubSubInfos = this .updateComplianceStatus(complianceChangeInfo, recordsMetadata, output); this.recordsRepo.createOrUpdate(recordsMetadata); StringBuilder recordsId = new StringBuilder(); @@ -72,9 +89,9 @@ public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeS recordsId.append(", ").append(recordMetadata.getId()); } this.auditLogger.updateRecordsComplianceStateSuccess( - singletonList("[" + recordsId.toString().substring(2) + "]")); + singletonList("[" + recordsId.substring(2) + "]")); - this.pubSubclient.publishMessage(headers, pubsubInfos); + this.pubSubClient.publishMessage(headers, pubSubInfos); } } while (cursor != null); } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java index f55db10a..4db9ec80 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.messagebus; public interface IMessageFactory { @@ -9,5 +26,4 @@ public interface IMessageFactory { void sendMessage(String msg); void sendMessage(String queueName, String msg); - } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java index a38f98f1..0038bb9e 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.model; import java.util.ArrayList; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java index 8a010588..65bfe11a 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.model; import java.util.Map; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java index f96066c8..ae2ea414 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.model; import java.util.List; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java index 30836e90..accf7016 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java @@ -1,20 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.persistence; import com.mongodb.client.MongoCollection; import org.bson.Document; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.storage.provider.reference.util.MongoClientHandler; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class MongoDdmsClient { - @Autowired private MongoClientHandler mongoClientHandler; - @Autowired - private TenantInfo tenantInfo; + public MongoDdmsClient(MongoClientHandler mongoClientHandler) { + this.mongoClientHandler = mongoClientHandler; + } public MongoCollection getMongoCollection(String dbName, String collectionName) { return mongoClientHandler.getMongoClient().getDatabase(dbName) diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java index 9f03d474..8c165109 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.repository; import static com.mongodb.client.model.Filters.eq; @@ -9,6 +26,7 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.bson.Document; import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; @@ -20,15 +38,12 @@ import org.springframework.stereotype.Repository; @Repository public class QueryRepositoryImpl implements IQueryRepository { + private MongoDdmsClient mongoDdmsClient; @Autowired - private IRecordsMetadataRepository recordsMetadataRepository; - - @Autowired - private ISchemaRepository schemaRepository; - - @Autowired - private MongoDdmsClient mongoDdmsClient; + public QueryRepositoryImpl(MongoDdmsClient mongoDdmsClient) { + this.mongoDdmsClient = mongoDdmsClient; + } @Override public DatastoreQueryResult getAllKinds(Integer limit, String cursor) { @@ -53,7 +68,7 @@ public class QueryRepositoryImpl implements IQueryRepository { boolean paginated = false; int numRecords = PAGE_SIZE; - if (limit != null) { + if (Objects.nonNull(limit)) { numRecords = limit > 0 ? limit : PAGE_SIZE; paginated = true; } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java index 88eb82d0..b69f712c 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.repository; import static com.mongodb.client.model.Filters.eq; @@ -27,15 +44,18 @@ import org.springframework.stereotype.Repository; public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository { public static final String STORAGE_RECORD = "StorageRecord"; + private final MongoDdmsClient mongoDdmsClient; @Autowired - private MongoDdmsClient mongoDdmsClient; + public RecordsMetadataRepositoryImpl(MongoDdmsClient mongoDdmsClient) { + this.mongoDdmsClient = mongoDdmsClient; + } @Override public List createOrUpdate(List recordsMetadata) { MongoCollection mongoCollection = mongoDdmsClient .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); - if (recordsMetadata != null) { + if (Objects.nonNull(recordsMetadata)) { for (RecordMetadata recordMetadata : recordsMetadata) { RecordMetadataDocument recordMetadataDocument = convertToRecordMetadataDocument( recordMetadata); @@ -109,7 +129,8 @@ public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository } @Override - public AbstractMap.SimpleEntry> queryByLegal(String legalTagName, LegalCompliance status, int limit) { + public AbstractMap.SimpleEntry> queryByLegal(String legalTagName, + LegalCompliance status, int limit) { return null; } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java index b9ba1012..c8ae6fd7 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.repository; import static com.mongodb.client.model.Filters.eq; @@ -26,9 +43,13 @@ public class SchemaRepositoryImpl implements ISchemaRepository { public static final String RECORD_STORAGE = "StorageRecord"; public static final String SCHEMA_DATABASE = "schema"; - @Autowired private MongoDdmsClient mongoClient; + @Autowired + public SchemaRepositoryImpl(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } + @Override public void add(Schema schema, String user) { MongoCollection collection = this.mongoClient diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java index 2482333c..ff2a17f6 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.security; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java index 15300d52..76eba1e2 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.security; import org.springframework.security.core.Authentication; diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java index 56fa7d97..7d073db5 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java @@ -1,22 +1,43 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.service; import static java.util.Collections.singletonList; -import javax.inject.Inject; import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; import org.opengroup.osdu.storage.logging.StorageAuditLogger; import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; import org.opengroup.osdu.storage.service.BatchServiceImpl; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Service public class BatchServiceReferenceImpl extends BatchServiceImpl { - @Inject - private StorageAuditLogger auditLogger; + private final StorageAuditLogger auditLogger; + private final IQueryRepository queryRepository; - @Inject - private IQueryRepository queryRepository; + @Autowired + public BatchServiceReferenceImpl(StorageAuditLogger auditLogger, + IQueryRepository queryRepository) { + this.auditLogger = auditLogger; + this.queryRepository = queryRepository; + } @Override public DatastoreQueryResult getAllKinds(String cursor, Integer limit) { diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java index 390996e4..409e0134 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.service; import com.google.gson.Gson; @@ -14,8 +31,12 @@ import org.springframework.stereotype.Component; @Component public class MessageBusImpl implements IMessageBus { + private final IMessageFactory messageQueue; + @Autowired - private IMessageFactory messageQueue; + public MessageBusImpl(IMessageFactory messageQueue) { + this.messageQueue = messageQueue; + } public void publishMessage(DpsHeaders headers, PubSubInfo... messages) { final int BATCH_SIZE = 50; @@ -31,7 +52,7 @@ public class MessageBusImpl implements IMessageBus { message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); headers.addCorrelationIdIfMissing(); message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); - message.put(DpsHeaders.AUTHORIZATION,headers.getAuthorization()); + message.put(DpsHeaders.AUTHORIZATION, headers.getAuthorization()); messageQueue.sendMessage(gson.toJson(message)); } } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java index 976463c8..7bbb0a9a 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.util; import com.mongodb.ConnectionString; @@ -6,9 +23,11 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; +import org.opengroup.osdu.storage.provider.reference.config.MongoDBConfigProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component @@ -19,18 +38,7 @@ public class MongoClientHandler { private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; private com.mongodb.client.MongoClient mongoClient = null; - - @Value("${mongo.db.url:#{null}}") - private String dbUrl; - - @Value("${mongo.db.apikey:#{null}}") - private String apiKey; - - @Value("${mongo.db.user:#{null}}") - private String dbUser; - - @Value("${mongo.db.password:#{null}}") - private String dbPassword; + private MongoDBConfigProperties mongoDBConfigProperties; private MongoClient getOrInitMongoClient() throws RuntimeException { if (mongoClient != null) { @@ -39,9 +47,9 @@ public class MongoClientHandler { final String connectionString = String.format("%s%s:%s@%s/?%s", MONGO_PREFIX, - dbUser, - dbPassword, - dbUrl, + mongoDBConfigProperties.getMongoDbUser(), + mongoDBConfigProperties.getMongoDbPassword(), + mongoDBConfigProperties.getMongoDbUrl(), MONGO_OPTIONS); ConnectionString connString = new ConnectionString(connectionString); MongoClientSettings settings = MongoClientSettings.builder() @@ -65,4 +73,8 @@ public class MongoClientHandler { return mongoClient; } + @Autowired + public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) { + this.mongoDBConfigProperties = mongoDBConfigProperties; + } } diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java index ae5153b9..07adc6bc 100644 --- a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.util; import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; @@ -7,8 +24,9 @@ import org.springframework.web.context.annotation.RequestScope; @Component @RequestScope public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { - @Override - public String getIdToken(String tenantName){ - return "dont-have-one"; - } + + @Override + public String getIdToken(String tenantName) { + return "dont-have-one"; + } } diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties index d459d529..1e7c8dc1 100644 --- a/provider/storage-reference/src/main/resources/application.properties +++ b/provider/storage-reference/src/main/resources/application.properties @@ -8,20 +8,20 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 LEGALTAG_API=https://os-legal-ibm/api/legal/v1 -mongo.db.url=localhost:27017 -mongo.db.user=admin -mongo.db.password=admin +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= #amqp://guest:guest@127.0.0.1:5672/%2F by default -mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672/%2F -minio.endpoint.url=http://127.0.0.1:9000 -minio.access.key=adminadmin -minio.secret.key=adminadmin -minio.region=admin -minio.prefix=local-dev -minio.bucket.record.name=record-bucket +minio-endpoint-url=http://127.0.0.1:9000 +minio-access-key= +minio-secret-key= +minio-region= +minio-prefix=local-dev +minio-bucket-record-name=record-bucket -gcp.redis.host=localhost -gcp.redis.port=6379 -gcp.redis.exp.time=10 \ No newline at end of file +gcp-redis-host=localhost +gcp-redis-port=6379 +gcp-redis-exp-time=10 \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java index 1ad3548f..f4fb1e2f 100644 --- a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java @@ -1,9 +1,27 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference; import static org.mockito.Matchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.MockitoAnnotations.initMocks; +import static org.mockito.Mockito.when; + import io.minio.MinioClient; import java.util.HashMap; import java.util.Map; @@ -14,17 +32,22 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.storage.Record; import org.opengroup.osdu.core.common.model.storage.RecordData; import org.opengroup.osdu.core.common.model.storage.RecordMetadata; import org.opengroup.osdu.core.common.model.storage.RecordProcessing; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; @RunWith(MockitoJUnitRunner.class) public class CloudStorageImplTest extends TestCase { + private RecordProcessing[] recordProcessingArray = new RecordProcessing[1]; + private RecordProcessing recordProcessing; + private RecordMetadata recordMetadata; + private String bucketName = "osdu-sample-osdu-file"; + @Mock private CloudObjectStorageFactory factory; @@ -32,19 +55,13 @@ public class CloudStorageImplTest extends TestCase { private MinioClient minioClient; @Mock - private JaxRsDpsLog log; + private MinIoConfigProperties minIoConfigProperties; @InjectMocks private CloudStorageImpl cloudStorage; - private RecordProcessing[] recordProcessings = new RecordProcessing[1]; - private RecordProcessing recordProcessing; - private RecordMetadata recordMetadata; - @Before public void setup() { - initMocks(this); - recordMetadata = new RecordMetadata(); recordMetadata.setKind("test-record-id"); recordMetadata.setId("test-record-id"); @@ -62,39 +79,51 @@ public class CloudStorageImplTest extends TestCase { RecordData recordData = new RecordData(record); recordProcessing.setRecordData(recordData); - recordProcessings[0] = recordProcessing; - cloudStorage.setRecordBucketName("record-bucket"); + recordProcessingArray[0] = recordProcessing; } @Test public void write_test() throws Exception { - cloudStorage.write(recordProcessings); + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + cloudStorage.write(recordProcessingArray); + verify(minioClient, times(1)).putObject(any()); } @Test public void delete_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + cloudStorage.delete(recordMetadata); + verify(minioClient, times(1)).removeObject(any()); } @Test public void deleteVersion_test() throws Exception { - cloudStorage.deleteVersion(recordMetadata, (long) 1); + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + cloudStorage.deleteVersion(recordMetadata, 1L); + verify(minioClient, times(1)).removeObject(any()); } @Test(expected = AppException.class) public void read_with_version_test() throws Exception { - cloudStorage.read(recordMetadata, (long) 1, false); + cloudStorage.read(recordMetadata, 1L, false); + verify(minioClient, times(1)).getObject(any()); } @Test public void read_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + Map map = new HashMap<>(); map.put("common:welldb:1", "opendes:ds:mytest1:1.0.0/common:welldb:123456/1603618609515093"); cloudStorage.read(map); + verify(minioClient, times(1)).getObject(any()); } } \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java index 88c8c0d5..5fd9839c 100644 --- a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java @@ -1,3 +1,20 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opengroup.osdu.storage.provider.reference.context; import org.junit.Ignore; @@ -5,13 +22,14 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; + @Ignore @RunWith(SpringRunner.class) @SpringBootTest public class ProviderSpringContextTest { @Test - public void load_spring_context(){ + public void load_spring_context() { } } -- GitLab From 8271ea214f20654054a481495748fe5aa6dd7f52 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Tue, 16 Feb 2021 11:50:43 +0300 Subject: [PATCH 23/44] Cannot retrieve a record with space --- .../org/opengroup/osdu/storage/api/RecordApi.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java index 17719bd0..3051f9c6 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java @@ -53,6 +53,7 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope @Validated public class RecordApi { + private static final String RECORD_ID_REGEX = "^[\\w\\-\\.]+:[\\w-\\.]+:[\\w\\-\\.\\:]+$"; @Autowired private DpsHeaders headers; @@ -81,14 +82,14 @@ public class RecordApi { @GetMapping("/versions/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getRecordVersions( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { return new ResponseEntity(this.queryService.listVersions(id), HttpStatus.OK); } @DeleteMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.ADMIN + "')") - public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { this.recordService.purgeRecord(id); return new ResponseEntity(HttpStatus.NO_CONTENT); @@ -96,7 +97,7 @@ public class RecordApi { @PostMapping("/{id}:delete") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") - public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { this.recordService.deleteRecord(id, this.headers.getUserEmail()); return new ResponseEntity(HttpStatus.NO_CONTENT); @@ -105,7 +106,7 @@ public class RecordApi { @GetMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getLatestRecordVersion( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id, @RequestParam(name = "attribute", required = false) String[] attributes) { return new ResponseEntity(this.queryService.getRecordInfo(id, attributes), HttpStatus.OK); @@ -114,7 +115,7 @@ public class RecordApi { @GetMapping("/{id}/{version}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getSpecificRecordVersion( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id, @PathVariable("version") long version, @RequestParam(name = "attribute", required = false) String[] attributes) { -- GitLab From ba4851078981b5a3efed10991f27cd2e79574c41 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 16 Feb 2021 15:20:47 +0400 Subject: [PATCH 24/44] GONRG-1766: Implement logic for Storage [publish] https://jiraeu.epam.com/browse/GONRG-1766 * Added logback.xml * Updated pom.xml --- provider/storage-gcp/pom.xml | 20 +++++++++++- .../src/main/resources/logback.xml | 32 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 provider/storage-gcp/src/main/resources/logback.xml diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 628fea91..444b20c7 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -40,7 +40,7 @@ org.opengroup.osdu core-lib-gcp - 0.3.25 + 0.6.0-SNAPSHOT @@ -54,6 +54,18 @@ 0.6.0-SNAPSHOT + + ch.qos.logback.contrib + logback-json-classic + 0.1.5 + + + + ch.qos.logback.contrib + logback-jackson + 0.1.5 + + com.google.cloud google-cloud-storage @@ -111,6 +123,12 @@ + + org.opengroup.osdu.storage.StorageApplication + + + c:\Users\Artem_Dobrynin\GO3-NRG\OsduCreds\osdu-gcp-sa-epam.json + diff --git a/provider/storage-gcp/src/main/resources/logback.xml b/provider/storage-gcp/src/main/resources/logback.xml new file mode 100644 index 00000000..5e5ac6a0 --- /dev/null +++ b/provider/storage-gcp/src/main/resources/logback.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + yyyy-MM-dd HH:mm:ss.SSS + Etc/UTC + true + + + false + + + + + + + + + + + \ No newline at end of file -- GitLab From 086677a33177705e0fb106a35afdd541402a2b4e Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 16 Feb 2021 17:55:15 +0400 Subject: [PATCH 25/44] GONRG-1766: Implement logic for Storage [publish] https://jiraeu.epam.com/browse/GONRG-1766 * Added logback.xml * Updated pom.xml --- provider/storage-gcp/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 444b20c7..2bb1641e 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -123,12 +123,6 @@ - - org.opengroup.osdu.storage.StorageApplication - - - c:\Users\Artem_Dobrynin\GO3-NRG\OsduCreds\osdu-gcp-sa-epam.json - -- GitLab From bf4ced5b1486e8e8a030c639ae88937eb93372c5 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Tue, 16 Feb 2021 19:21:22 +0400 Subject: [PATCH 26/44] GONRG-1766: Implement logic for Storage [publish] https://jiraeu.epam.com/browse/GONRG-1766 * Added logback.xml * Updated pom.xml --- provider/storage-gcp/pom.xml | 6 ------ provider/storage-gcp/src/main/resources/logback.xml | 10 +++++++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 2bb1641e..1d866e40 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -60,12 +60,6 @@ 0.1.5 - - ch.qos.logback.contrib - logback-jackson - 0.1.5 - - com.google.cloud google-cloud-storage diff --git a/provider/storage-gcp/src/main/resources/logback.xml b/provider/storage-gcp/src/main/resources/logback.xml index 5e5ac6a0..e6bb61f9 100644 --- a/provider/storage-gcp/src/main/resources/logback.xml +++ b/provider/storage-gcp/src/main/resources/logback.xml @@ -2,8 +2,12 @@ - - + + + %yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n + utf8 + + @@ -17,7 +21,7 @@ Etc/UTC true - + false -- GitLab From 5a71a0878c2e08d1c18c520ce36c2aea2e5e622b Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Fri, 19 Feb 2021 11:08:04 +0000 Subject: [PATCH 27/44] Revert "Cannot retrieve a record with space" This reverts commit 8271ea214f20654054a481495748fe5aa6dd7f52 --- .../org/opengroup/osdu/storage/api/RecordApi.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java index 3051f9c6..17719bd0 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java @@ -53,7 +53,6 @@ import org.springframework.web.context.annotation.RequestScope; @RequestScope @Validated public class RecordApi { - private static final String RECORD_ID_REGEX = "^[\\w\\-\\.]+:[\\w-\\.]+:[\\w\\-\\.\\:]+$"; @Autowired private DpsHeaders headers; @@ -82,14 +81,14 @@ public class RecordApi { @GetMapping("/versions/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getRecordVersions( - @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { return new ResponseEntity(this.queryService.listVersions(id), HttpStatus.OK); } @DeleteMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.ADMIN + "')") - public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, + public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { this.recordService.purgeRecord(id); return new ResponseEntity(HttpStatus.NO_CONTENT); @@ -97,7 +96,7 @@ public class RecordApi { @PostMapping("/{id}:delete") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") - public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, + public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id) { this.recordService.deleteRecord(id, this.headers.getUserEmail()); return new ResponseEntity(HttpStatus.NO_CONTENT); @@ -106,7 +105,7 @@ public class RecordApi { @GetMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getLatestRecordVersion( - @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id, @RequestParam(name = "attribute", required = false) String[] attributes) { return new ResponseEntity(this.queryService.getRecordInfo(id, attributes), HttpStatus.OK); @@ -115,7 +114,7 @@ public class RecordApi { @GetMapping("/{id}/{version}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getSpecificRecordVersion( - @PathVariable("id") @Pattern(regexp = RECORD_ID_REGEX, + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, message = ValidationDoc.INVALID_RECORD_ID) String id, @PathVariable("version") long version, @RequestParam(name = "attribute", required = false) String[] attributes) { -- GitLab From 05cc444f46de408c4ced049e2665c893299734c7 Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Fri, 19 Feb 2021 19:48:01 +0400 Subject: [PATCH 28/44] GONRG-1766: Implement logic for Storage [publish] https://jiraeu.epam.com/browse/GONRG-1766 * Added logback.xml * Updated pom.xml --- provider/storage-gcp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-gcp/pom.xml b/provider/storage-gcp/pom.xml index 1d866e40..f6d8745b 100644 --- a/provider/storage-gcp/pom.xml +++ b/provider/storage-gcp/pom.xml @@ -40,7 +40,7 @@ org.opengroup.osdu core-lib-gcp - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT -- GitLab From 6b39e3f4366715e89517fc0ed914288056514f58 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Thu, 25 Feb 2021 15:33:20 +0400 Subject: [PATCH 29/44] Cannot retrieve a record with space in id --- .../opengroup/osdu/storage/service/IngestionServiceImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java index 7b51eb0d..49644b15 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java @@ -136,6 +136,10 @@ public class IngestionServiceImpl implements IngestionService { throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid record id", msg); } + if (id.contains("%20")) { + throw new AppException(HttpStatus.SC_BAD_REQUEST, "Validation error.", + "Not a valid record id. Found: " + id); + } ids.add(id); } else { record.createNewRecordId(tenantName, record.getKind()); -- GitLab From 174eb6b684e6983e60bb713fe84ae35df5ce17fd Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Fri, 26 Feb 2021 13:14:13 +0400 Subject: [PATCH 30/44] add validation in to controller --- .../opengroup/osdu/storage/api/RecordApi.java | 38 +++++++++++-------- .../storage/service/IngestionServiceImpl.java | 4 -- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java index 17719bd0..03e37c8b 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java @@ -14,13 +14,14 @@ package org.opengroup.osdu.storage.api; +import java.nio.charset.Charset; import java.util.List; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.Pattern; import javax.validation.constraints.Size; +import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -47,6 +48,7 @@ import org.opengroup.osdu.storage.service.QueryService; import org.opengroup.osdu.storage.service.RecordService; import org.opengroup.osdu.core.common.model.storage.validation.ValidationDoc; import org.springframework.web.context.annotation.RequestScope; +import org.springframework.web.util.UriUtils; @RestController @RequestMapping("records") @@ -81,43 +83,47 @@ public class RecordApi { @GetMapping("/versions/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getRecordVersions( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, - message = ValidationDoc.INVALID_RECORD_ID) String id) { - return new ResponseEntity(this.queryService.listVersions(id), HttpStatus.OK); + @PathVariable("id") String id) { + return new ResponseEntity(this.queryService.listVersions(getValidId(id)), HttpStatus.OK); } @DeleteMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.ADMIN + "')") - public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, - message = ValidationDoc.INVALID_RECORD_ID) String id) { - this.recordService.purgeRecord(id); + public ResponseEntity purgeRecord(@PathVariable("id") String id) { + this.recordService.purgeRecord(getValidId(id)); return new ResponseEntity(HttpStatus.NO_CONTENT); } @PostMapping("/{id}:delete") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") - public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, - message = ValidationDoc.INVALID_RECORD_ID) String id) { - this.recordService.deleteRecord(id, this.headers.getUserEmail()); + public ResponseEntity deleteRecord(@PathVariable("id") String id) { + this.recordService.deleteRecord(getValidId(id), this.headers.getUserEmail()); return new ResponseEntity(HttpStatus.NO_CONTENT); } @GetMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getLatestRecordVersion( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, - message = ValidationDoc.INVALID_RECORD_ID) String id, + @PathVariable("id") String id, @RequestParam(name = "attribute", required = false) String[] attributes) { - return new ResponseEntity(this.queryService.getRecordInfo(id, attributes), HttpStatus.OK); + return new ResponseEntity(this.queryService.getRecordInfo(getValidId(id), attributes), HttpStatus.OK); } @GetMapping("/{id}/{version}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getSpecificRecordVersion( - @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, - message = ValidationDoc.INVALID_RECORD_ID) String id, + @PathVariable("id") String id, @PathVariable("version") long version, @RequestParam(name = "attribute", required = false) String[] attributes) { - return new ResponseEntity(this.queryService.getRecordInfo(id, version, attributes), HttpStatus.OK); + return new ResponseEntity(this.queryService.getRecordInfo(getValidId(id), version, attributes), HttpStatus.OK); + } + + private String getValidId(String id) { + String encodedId = UriUtils.encode(id, Charset.defaultCharset()).replace("%3A", ":"); + if (!encodedId.matches(ValidationDoc.RECORD_ID_REGEX)) { + throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Validation error.", + "Not a valid record id. Found: " + encodedId); + } + return encodedId; } } \ No newline at end of file diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java index 49644b15..7b51eb0d 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/IngestionServiceImpl.java @@ -136,10 +136,6 @@ public class IngestionServiceImpl implements IngestionService { throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid record id", msg); } - if (id.contains("%20")) { - throw new AppException(HttpStatus.SC_BAD_REQUEST, "Validation error.", - "Not a valid record id. Found: " + id); - } ids.add(id); } else { record.createNewRecordId(tenantName, record.getKind()); -- GitLab From 9bbdcb1d44f362c8892ef53f1836a0e7d022adc3 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Mon, 1 Mar 2021 10:55:51 +0400 Subject: [PATCH 31/44] fix unit tests --- .../opengroup/osdu/storage/api/RecordApiTest.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/storage-core/src/test/java/org/opengroup/osdu/storage/api/RecordApiTest.java b/storage-core/src/test/java/org/opengroup/osdu/storage/api/RecordApiTest.java index 7b83eeae..38d4ee2d 100644 --- a/storage-core/src/test/java/org/opengroup/osdu/storage/api/RecordApiTest.java +++ b/storage-core/src/test/java/org/opengroup/osdu/storage/api/RecordApiTest.java @@ -51,6 +51,7 @@ public class RecordApiTest { private final String USER = "user"; private final String TENANT = "tenant1"; + private final String RECORD_ID = "osdu:anyID:any"; @Mock private IngestionService ingestionService; @@ -125,8 +126,6 @@ public class RecordApiTest { @Test public void should_returnHttp200_when_gettingRecordVersionsSuccessfully() { - final String RECORD_ID = "anyID"; - List versions = new ArrayList(); versions.add(1L); versions.add(2L); @@ -149,8 +148,6 @@ public class RecordApiTest { @Test public void should_returnHttp204_when_purgingRecordSuccessfully() { - final String RECORD_ID = "anyID"; - ResponseEntity response = this.sut.purgeRecord(RECORD_ID); assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatusCodeValue()); @@ -158,8 +155,6 @@ public class RecordApiTest { @Test public void should_returnHttp200_when_gettingTheLatestVersionOfARecordSuccessfully() { - final String RECORD_ID = "anyID"; - when(this.queryService.getRecordInfo(RECORD_ID, new String[] {})).thenReturn(RECORD_ID); ResponseEntity response = this.sut.getLatestRecordVersion(RECORD_ID, new String[] {}); @@ -172,10 +167,9 @@ public class RecordApiTest { @Test public void should_returnHttp200_when_gettingSpecificVersionOfARecordSuccessfully() { - final String RECORD_ID = "anyID"; final long VERSION = 1L; - String expectedRecord = "{\"id\": \"anyID\",\r\n\"version\": 1}"; + String expectedRecord = "{\"id\": \"osdu:anyID:any\",\r\n\"version\": 1}"; when(this.queryService.getRecordInfo(RECORD_ID, VERSION, new String[] {})).thenReturn(expectedRecord); @@ -190,8 +184,6 @@ public class RecordApiTest { @Test public void should_returnHttp204_when_deletingRecordSuccessfully() { - final String RECORD_ID = "anyID"; - ResponseEntity response = this.sut.deleteRecord(RECORD_ID); assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatusCodeValue()); -- GitLab From cf40674ff4a46a1e7bd426ca5a8258cc740c02dd Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Mon, 1 Mar 2021 11:14:18 +0400 Subject: [PATCH 32/44] GONRG-1756: STORAGE: Security headers are malformed and even those are not set https://jiraeu.epam.com/browse/GONRG-1756 * Updated headers for Storage requests --- provider/storage-gcp/src/main/resources/logback.xml | 7 ++++--- .../org/opengroup/osdu/storage/util/StorageFilter.java | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/provider/storage-gcp/src/main/resources/logback.xml b/provider/storage-gcp/src/main/resources/logback.xml index e6bb61f9..3d4fbc03 100644 --- a/provider/storage-gcp/src/main/resources/logback.xml +++ b/provider/storage-gcp/src/main/resources/logback.xml @@ -1,6 +1,7 @@ + @@ -8,7 +9,7 @@ utf8 - + @@ -28,9 +29,9 @@ - + - \ No newline at end of file + diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java index 8bf3b5dc..721a444f 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.ServletException; @@ -61,7 +62,6 @@ public class StorageFilter implements Filter { this.dpsHeaders.put(FOR_HEADER_NAME, fetchConversionHeader); } - chain.doFilter(request, response); HttpServletResponse httpResponse = (HttpServletResponse) response; @@ -69,7 +69,8 @@ public class StorageFilter implements Filter { Map> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; for (Map.Entry> header : standardHeaders.entrySet()) { - httpResponse.addHeader(header.getKey(), header.getValue().toString()); + httpResponse.addHeader(header.getKey(), header.getValue().stream().map(o -> o.toString()).collect( + Collectors.joining(" "))); } httpResponse.addHeader(DpsHeaders.CORRELATION_ID, this.dpsHeaders.getCorrelationId()); @@ -78,6 +79,9 @@ public class StorageFilter implements Filter { if (httpRequest.getMethod().equalsIgnoreCase(OPTIONS_STRING)) { httpResponse.setStatus(HttpStatus.SC_OK); } + + chain.doFilter(request, response); + } @Override -- GitLab From 020b9924454a3fd8d9273bd03d2e8f4c9bb1d7fe Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Mon, 1 Mar 2021 11:34:33 +0400 Subject: [PATCH 33/44] GONRG-1756: STORAGE: Security headers are malformed and even those are not set https://jiraeu.epam.com/browse/GONRG-1756 * Updated headers for Storage requests --- .../java/org/opengroup/osdu/storage/util/StorageFilter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java index 721a444f..a85de4c0 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java @@ -69,8 +69,7 @@ public class StorageFilter implements Filter { Map> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS; for (Map.Entry> header : standardHeaders.entrySet()) { - httpResponse.addHeader(header.getKey(), header.getValue().stream().map(o -> o.toString()).collect( - Collectors.joining(" "))); + httpResponse.addHeader(header.getKey(), header.getValue().toString()); } httpResponse.addHeader(DpsHeaders.CORRELATION_ID, this.dpsHeaders.getCorrelationId()); -- GitLab From df2cb0eba60383dc23d036941a1d63af0468cd1b Mon Sep 17 00:00:00 2001 From: Artem Dobrynin Date: Mon, 1 Mar 2021 13:47:10 +0400 Subject: [PATCH 34/44] GONRG-1756: STORAGE: Security headers are malformed and even those are not set https://jiraeu.epam.com/browse/GONRG-1756 * Updated headers for Storage requests --- provider/storage-gcp/src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-gcp/src/main/resources/application.properties b/provider/storage-gcp/src/main/resources/application.properties index 57dda0d9..bff7492f 100644 --- a/provider/storage-gcp/src/main/resources/application.properties +++ b/provider/storage-gcp/src/main/resources/application.properties @@ -3,7 +3,7 @@ LOG_PREFIX=storage osdu.gcp.storage.gcs.enable-impersonalization=false server.servlet.contextPath=/api/storage/v2/ -logging.level.org.springframework.web=DEBUG +logging.level.org.springframework.web={LOG_LEVEL:DEBUG} server.port=8080 JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 -- GitLab From 2157e2b5773df8b061e8391192844f85ec08e68d Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Wed, 3 Mar 2021 15:09:41 +0400 Subject: [PATCH 35/44] fix webSecurityConfig, update README --- provider/storage-gcp/README.md | 13 +++++++ .../gcp/security/GSuiteSecurityConfig.java | 17 +++++++++ .../opengroup/osdu/storage/api/RecordApi.java | 38 ++++++++----------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/provider/storage-gcp/README.md b/provider/storage-gcp/README.md index 3bca0664..93d2ccaa 100644 --- a/provider/storage-gcp/README.md +++ b/provider/storage-gcp/README.md @@ -28,6 +28,19 @@ In order to run the service locally or remotely, you will need to have the follo | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +###Requirements for requests. +Record identifiers cannot contain a space character. +At the same time, they may contain a % character, which, when combined with subsequent numeric characters, +may cause the application to misinterpret that combination. +For example, the "%20" combination will be interpreted as a space " " character. +To correctly transfer such an identifier, you should additionally perform the url-encode operation on it. +This functionality can be built into the front-end application, or you can use an online url-encoder +tool ( eg.: https://www.urlencoder.org/). +Thus, having ID "osdu:work-product-component--WellboreMarkerSet:3D%20Kirchhoff%20DepthMigration" (with %20 combination) +you should url-encode it and request +"osdu%3Awork-product-component--WellboreMarkerSet%3A3D%2520Kirchhoff%2520DepthMigration" instead. + + ### Run Locally Check that maven is installed: diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java index 60914fe7..ebe9a4f8 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java @@ -14,10 +14,14 @@ package org.opengroup.osdu.storage.provider.gcp.security; +import org.springframework.context.annotation.Bean; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.web.firewall.HttpFirewall; +import org.springframework.security.web.firewall.StrictHttpFirewall; @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) @@ -29,4 +33,17 @@ public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { .httpBasic().disable() .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy } + + @Override + public void configure(WebSecurity web) throws Exception { + super.configure(web); + web.httpFirewall(allowUrlEncodedSlashHttpFirewall()); + } + + @Bean + public HttpFirewall allowUrlEncodedSlashHttpFirewall() { + StrictHttpFirewall firewall = new StrictHttpFirewall(); + firewall.setAllowUrlEncodedPercent(true); + return firewall; + } } diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java index 090bc159..41be38ba 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/api/RecordApi.java @@ -14,14 +14,13 @@ package org.opengroup.osdu.storage.api; -import java.nio.charset.Charset; import java.util.List; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; import javax.validation.constraints.Size; -import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.storage.Record; import org.opengroup.osdu.core.common.model.storage.RecordVersions; @@ -49,7 +48,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.annotation.RequestScope; -import org.springframework.web.util.UriUtils; @RestController @RequestMapping("records") @@ -85,47 +83,43 @@ public class RecordApi { @GetMapping("/versions/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getRecordVersions( - @PathVariable("id") String id) { - return new ResponseEntity(this.queryService.listVersions(getValidId(id)), HttpStatus.OK); + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + message = ValidationDoc.INVALID_RECORD_ID) String id) { + return new ResponseEntity(this.queryService.listVersions(id), HttpStatus.OK); } @DeleteMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.ADMIN + "')") - public ResponseEntity purgeRecord(@PathVariable("id") String id) { - this.recordService.purgeRecord(getValidId(id)); + public ResponseEntity purgeRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + message = ValidationDoc.INVALID_RECORD_ID) String id) { + this.recordService.purgeRecord(id); return new ResponseEntity(HttpStatus.NO_CONTENT); } @PostMapping("/{id}:delete") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") - public ResponseEntity deleteRecord(@PathVariable("id") String id) { - this.recordService.deleteRecord(getValidId(id), this.headers.getUserEmail()); + public ResponseEntity deleteRecord(@PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + message = ValidationDoc.INVALID_RECORD_ID) String id) { + this.recordService.deleteRecord(id, this.headers.getUserEmail()); return new ResponseEntity(HttpStatus.NO_CONTENT); } @GetMapping("/{id}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getLatestRecordVersion( - @PathVariable("id") String id, + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + message = ValidationDoc.INVALID_RECORD_ID) String id, @RequestParam(name = "attribute", required = false) String[] attributes) { - return new ResponseEntity(this.queryService.getRecordInfo(getValidId(id), attributes), HttpStatus.OK); + return new ResponseEntity(this.queryService.getRecordInfo(id, attributes), HttpStatus.OK); } @GetMapping("/{id}/{version}") @PreAuthorize("@authorizationFilter.hasRole('" + StorageRole.VIEWER + "', '" + StorageRole.CREATOR + "', '" + StorageRole.ADMIN + "')") public ResponseEntity getSpecificRecordVersion( - @PathVariable("id") String id, + @PathVariable("id") @Pattern(regexp = ValidationDoc.RECORD_ID_REGEX, + message = ValidationDoc.INVALID_RECORD_ID) String id, @PathVariable("version") long version, @RequestParam(name = "attribute", required = false) String[] attributes) { - return new ResponseEntity(this.queryService.getRecordInfo(getValidId(id), version, attributes), HttpStatus.OK); - } - - private String getValidId(String id) { - String encodedId = UriUtils.encode(id, Charset.defaultCharset()).replace("%3A", ":"); - if (!encodedId.matches(ValidationDoc.RECORD_ID_REGEX)) { - throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Validation error.", - "Not a valid record id. Found: " + encodedId); - } - return encodedId; + return new ResponseEntity(this.queryService.getRecordInfo(id, version, attributes), HttpStatus.OK); } } \ No newline at end of file -- GitLab From 9816e6c5d546e0c923c49bc0e8dde88b40e0e02e Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Fri, 5 Mar 2021 17:10:14 +0300 Subject: [PATCH 36/44] fix integration tests --- .../osdu/storage/records/UpdateRecordsMetadataTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java b/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java index 112dd7ee..6abe7162 100644 --- a/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java +++ b/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java @@ -209,17 +209,11 @@ public abstract class UpdateRecordsMetadataTest extends TestBase { JsonObject resultObject = bodyToJsonObject(updateResponse.getEntity(String.class)); assertEquals(RECORD_ID, resultObject.get("recordIds").getAsJsonArray().get(0).getAsString()); - resultObject = bodyToJsonObject(recordResponse.getEntity(String.class)); - assertEquals(TAG_VALUE1, resultObject.get("tags").getAsJsonObject().get(TAG_KEY).getAsString()); - //replace operation updateBody = buildUpdateTagBody(RECORD_ID, "replace", TAG_KEY + ":" + TAG_VALUE2); sendRequest("PATCH", "records", toJson(updateBody), testUtils.getToken()); recordResponse = sendRequest("GET", "records/" + RECORD_ID, EMPTY, testUtils.getToken()); - resultObject = bodyToJsonObject(recordResponse.getEntity(String.class)); - assertEquals(TAG_VALUE2, resultObject.get("tags").getAsJsonObject().get(TAG_KEY).getAsString()); - //remove operation updateBody = buildUpdateTagBody(RECORD_ID,"remove", TAG_KEY); sendRequest("PATCH", "records", toJson(updateBody), testUtils.getToken()); -- GitLab From 8150148f5e77c32d916e6a55ede1e675fd0d96f4 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Fri, 5 Mar 2021 18:32:49 +0300 Subject: [PATCH 37/44] fix integration tests --- .../osdu/storage/records/UpdateRecordsMetadataTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java b/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java index 6abe7162..112dd7ee 100644 --- a/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java +++ b/testing/storage-test-core/src/main/java/org/opengroup/osdu/storage/records/UpdateRecordsMetadataTest.java @@ -209,11 +209,17 @@ public abstract class UpdateRecordsMetadataTest extends TestBase { JsonObject resultObject = bodyToJsonObject(updateResponse.getEntity(String.class)); assertEquals(RECORD_ID, resultObject.get("recordIds").getAsJsonArray().get(0).getAsString()); + resultObject = bodyToJsonObject(recordResponse.getEntity(String.class)); + assertEquals(TAG_VALUE1, resultObject.get("tags").getAsJsonObject().get(TAG_KEY).getAsString()); + //replace operation updateBody = buildUpdateTagBody(RECORD_ID, "replace", TAG_KEY + ":" + TAG_VALUE2); sendRequest("PATCH", "records", toJson(updateBody), testUtils.getToken()); recordResponse = sendRequest("GET", "records/" + RECORD_ID, EMPTY, testUtils.getToken()); + resultObject = bodyToJsonObject(recordResponse.getEntity(String.class)); + assertEquals(TAG_VALUE2, resultObject.get("tags").getAsJsonObject().get(TAG_KEY).getAsString()); + //remove operation updateBody = buildUpdateTagBody(RECORD_ID,"remove", TAG_KEY); sendRequest("PATCH", "records", toJson(updateBody), testUtils.getToken()); -- GitLab From 43e39ba5200b7fb5592163d177f60e09b3a52de4 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Tue, 9 Mar 2021 10:26:56 +0300 Subject: [PATCH 38/44] fix integration tests --- .../DatastoreRecordsMetadataRepository.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java index bcca7a3c..c7048034 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java @@ -67,6 +67,7 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos public static final String MODIFY_USER = "modifyUser"; public static final String VERSION = "version"; public static final String STATUS = "status"; + public static final String TAGS = "tags"; public static final String LEGAL = "legal"; public static final String LEGAL_TAGS = "legaltags"; @@ -173,6 +174,7 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos + TimeUnit.NANOSECONDS.toMillis(entity.getTimestamp(CREATE_TIME).getNanos())); recordMetadata.setAcl(this.buildAclObject(entity.getEntity(ACL))); recordMetadata.setLegal(this.buildLegalObject(entity)); + recordMetadata.setTags(this.buildTags(entity.getEntity(TAGS))); if (entity.contains(ANCESTRY)) { RecordAncestry ancestry = new RecordAncestry(); @@ -217,6 +219,10 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos .set(CREATE_TIME, Timestamp.ofTimeMicroseconds(TimeUnit.MILLISECONDS.toMicros(record.getCreateTime()))) .set(LEGAL, this.buildLegalEntity(record.getLegal())); + if (record.getTags() != null) { + entityBuilder.set(TAGS, this.buildTagEntity(record.getTags())); + } + if (record.getAncestry() != null && !Collections.isEmpty(record.getAncestry().getParents())) { entityBuilder.set(ANCESTRY, FullEntity.newBuilder() .set(ANCESTRY_PARENTS, this.buildEntityArray(record.getAncestry().getParents())).build()); @@ -265,6 +271,22 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos } + private FullEntity buildTagEntity(Map tags) { + Builder keyBuilder = FullEntity.newBuilder(); + tags.entrySet().forEach(tag -> keyBuilder.set(tag.getKey(), tag.getValue())); + return keyBuilder.build(); + } + + private Map buildTags(FullEntity entity) { + if (entity != null) { + Map tags = new HashMap<>(); + entity.getProperties().entrySet().forEach(entityTag -> + tags.put(entityTag.getKey(), String.valueOf(entityTag.getValue()))); + return tags; + } + return null; + } + private String[] buildObjectArray(List> list) { String[] array = new String[list.size()]; -- GitLab From 1d1330c0772e3d1b84d5f9cbb92a7e0009beb0b7 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Fri, 9 Apr 2021 18:05:00 +0300 Subject: [PATCH 39/44] add new property, rollback previous changes --- .../gcp/security/GSuiteSecurityConfig.java | 17 ----------------- .../src/main/resources/application.properties | 5 ++++- .../osdu/storage/util/StorageFilter.java | 2 -- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java index ebe9a4f8..60914fe7 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/security/GSuiteSecurityConfig.java @@ -14,14 +14,10 @@ package org.opengroup.osdu.storage.provider.gcp.security; -import org.springframework.context.annotation.Bean; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.web.firewall.HttpFirewall; -import org.springframework.security.web.firewall.StrictHttpFirewall; @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) @@ -33,17 +29,4 @@ public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter { .httpBasic().disable() .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy } - - @Override - public void configure(WebSecurity web) throws Exception { - super.configure(web); - web.httpFirewall(allowUrlEncodedSlashHttpFirewall()); - } - - @Bean - public HttpFirewall allowUrlEncodedSlashHttpFirewall() { - StrictHttpFirewall firewall = new StrictHttpFirewall(); - firewall.setAllowUrlEncodedPercent(true); - return firewall; - } } diff --git a/provider/storage-gcp/src/main/resources/application.properties b/provider/storage-gcp/src/main/resources/application.properties index bff7492f..49270578 100644 --- a/provider/storage-gcp/src/main/resources/application.properties +++ b/provider/storage-gcp/src/main/resources/application.properties @@ -11,4 +11,7 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc PUBSUB_SEARCH_TOPIC=records-changed REDIS_GROUP_PORT=6379 -REDIS_STORAGE_PORT=6379 \ No newline at end of file +REDIS_STORAGE_PORT=6379 + +# Spring boot config +osdu.spring.config.enableEncodedPercent=true \ No newline at end of file diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java index 57303bb5..9923eb96 100644 --- a/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java +++ b/storage-core/src/main/java/org/opengroup/osdu/storage/util/StorageFilter.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import javax.inject.Inject; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -85,7 +84,6 @@ public class StorageFilter implements Filter { } chain.doFilter(request, response); - } @Override -- GitLab From fa8ad4918075eaea341757b7328ee437e222d94d Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Fri, 16 Apr 2021 15:45:02 +0300 Subject: [PATCH 40/44] add test.properties --- testing/storage-test-gcp/src/test/resources/test.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 testing/storage-test-gcp/src/test/resources/test.properties diff --git a/testing/storage-test-gcp/src/test/resources/test.properties b/testing/storage-test-gcp/src/test/resources/test.properties new file mode 100644 index 00000000..0f5f9384 --- /dev/null +++ b/testing/storage-test-gcp/src/test/resources/test.properties @@ -0,0 +1 @@ +enableEncodedPercentInURL=true \ No newline at end of file -- GitLab From 7f27bc772c40c5a768224cbcda270e855e80543a Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Mon, 19 Apr 2021 10:04:09 +0300 Subject: [PATCH 41/44] remove 'tags' --- .../storage/provider/gcp/DatastoreRecordsMetadataRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java index 697a9dbe..fddef353 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java @@ -81,7 +81,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos public static final String ANCESTRY = "ancestry"; public static final String ANCESTRY_PARENTS = "parents"; - public static final String TAGS = "tags"; @Autowired private IDatastoreFactory datastoreFactory; -- GitLab From 0fea1c358beded660379812d1a7cef2f2f9c3b0c Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Tue, 20 Apr 2021 18:13:14 +0300 Subject: [PATCH 42/44] bug fix in integration tests --- .../provider/gcp/DatastoreRecordsMetadataRepository.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java index fddef353..7b2d4a0a 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java @@ -192,14 +192,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos recordMetadata.setModifyUser(entity.getString(MODIFY_USER)); } - if (entity.contains(TAGS)) { - String tags = entity.getString(TAGS); - if (!tags.isEmpty()) { - Map tagsMap = new Gson().fromJson(tags, Map.class); - recordMetadata.setTags(tagsMap); - } - } - if (entity.contains(MODIFY_TIME)) { recordMetadata.setModifyTime(TimeUnit.SECONDS.toMillis(entity.getTimestamp(MODIFY_TIME).getSeconds()) + TimeUnit.NANOSECONDS.toMillis(entity.getTimestamp(MODIFY_TIME).getNanos())); -- GitLab From 9dfc4be96e74c8e81349fdc4f7e5b5fde023e483 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Wed, 21 Apr 2021 15:05:30 +0300 Subject: [PATCH 43/44] fix integration tests --- .../DatastoreRecordsMetadataRepository.java | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java index 7b2d4a0a..7fc4ba92 100644 --- a/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java +++ b/provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/DatastoreRecordsMetadataRepository.java @@ -72,7 +72,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos public static final String MODIFY_USER = "modifyUser"; public static final String VERSION = "version"; public static final String STATUS = "status"; - public static final String TAGS = "tags"; public static final String LEGAL = "legal"; public static final String LEGAL_TAGS = "legaltags"; @@ -81,6 +80,7 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos public static final String ANCESTRY = "ancestry"; public static final String ANCESTRY_PARENTS = "parents"; + public static final String TAGS = "tags"; @Autowired private IDatastoreFactory datastoreFactory; @@ -179,7 +179,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos + TimeUnit.NANOSECONDS.toMillis(entity.getTimestamp(CREATE_TIME).getNanos())); recordMetadata.setAcl(this.buildAclObject(entity.getEntity(ACL))); recordMetadata.setLegal(this.buildLegalObject(entity)); - recordMetadata.setTags(this.buildTags(entity.getEntity(TAGS))); if (entity.contains(ANCESTRY)) { RecordAncestry ancestry = new RecordAncestry(); @@ -192,6 +191,14 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos recordMetadata.setModifyUser(entity.getString(MODIFY_USER)); } + if (entity.contains(TAGS)) { + String tags = entity.getString(TAGS); + if (!tags.isEmpty()) { + Map tagsMap = new Gson().fromJson(tags, Map.class); + recordMetadata.setTags(tagsMap); + } + } + if (entity.contains(MODIFY_TIME)) { recordMetadata.setModifyTime(TimeUnit.SECONDS.toMillis(entity.getTimestamp(MODIFY_TIME).getSeconds()) + TimeUnit.NANOSECONDS.toMillis(entity.getTimestamp(MODIFY_TIME).getNanos())); @@ -224,10 +231,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos .set(CREATE_TIME, Timestamp.ofTimeMicroseconds(TimeUnit.MILLISECONDS.toMicros(record.getCreateTime()))) .set(LEGAL, this.buildLegalEntity(record.getLegal())); - if (record.getTags() != null) { - entityBuilder.set(TAGS, this.buildTagEntity(record.getTags())); - } - if (record.getAncestry() != null && !Collections.isEmpty(record.getAncestry().getParents())) { entityBuilder.set(ANCESTRY, FullEntity.newBuilder() .set(ANCESTRY_PARENTS, this.buildEntityArray(record.getAncestry().getParents())).build()); @@ -285,22 +288,6 @@ public class DatastoreRecordsMetadataRepository implements IRecordsMetadataRepos } - private FullEntity buildTagEntity(Map tags) { - Builder keyBuilder = FullEntity.newBuilder(); - tags.entrySet().forEach(tag -> keyBuilder.set(tag.getKey(), tag.getValue())); - return keyBuilder.build(); - } - - private Map buildTags(FullEntity entity) { - if (entity != null) { - Map tags = new HashMap<>(); - entity.getProperties().entrySet().forEach(entityTag -> - tags.put(entityTag.getKey(), String.valueOf(entityTag.getValue()))); - return tags; - } - return null; - } - private String[] buildObjectArray(List> list) { String[] array = new String[list.size()]; -- GitLab From f7fe32d1a1d6992964fec495a9830816ba506bf5 Mon Sep 17 00:00:00 2001 From: Igor_Filippov Date: Wed, 21 Apr 2021 15:56:20 +0300 Subject: [PATCH 44/44] fix integration tests --- .../osdu/storage/records/TestRecordsApiAcceptance.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/storage-test-gcp/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java b/testing/storage-test-gcp/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java index 727dfc79..66ba8664 100644 --- a/testing/storage-test-gcp/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java +++ b/testing/storage-test-gcp/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java @@ -18,6 +18,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.opengroup.osdu.storage.util.ConfigUtils; import org.opengroup.osdu.storage.util.GCPTestUtils; public class TestRecordsApiAcceptance extends RecordsApiAcceptanceTests { @@ -38,6 +39,7 @@ public class TestRecordsApiAcceptance extends RecordsApiAcceptanceTests { @Override public void setup() throws Exception { this.testUtils = new GCPTestUtils(); + this.configUtils = new ConfigUtils("test.properties"); } @After -- GitLab