From 291f7166f66910e4dbfdfc381111ccdfd46696ae Mon Sep 17 00:00:00 2001
From: "Riabokon Stanislav(EPAM)" <stanislav_riabokon@epam.com>
Date: Mon, 18 Jan 2021 14:45:02 -0500
Subject: [PATCH] GONRG-941 Added an audit event 'Query all kinds' for all
providers.
---
pom.xml | 1 +
provider/storage-reference/Dockerfile | 5 +
provider/storage-reference/docker-compose.yml | 12 +
.../deployment-os-storage-service.yml | 132 ++++++++++
provider/storage-reference/pom.xml | 116 ++++++++
.../provider/reference/CloudStorageImpl.java | 247 ++++++++++++++++++
.../app/StorageReferenceApplication.java | 34 +++
.../provider/reference/cache/GroupCache.java | 35 +++
.../reference/cache/LegalTagCache.java | 64 +++++
.../provider/reference/cache/SchemaCache.java | 30 +++
.../config/MinIoConfigProperties.java | 38 +++
.../config/MongoDBConfigProperties.java | 35 +++
.../config/RabbitMqConfigProperties.java | 32 +++
.../config/RedisConfigProperties.java | 34 +++
.../reference/di/RabbitMQFactoryImpl.java | 88 +++++++
.../reference/di/TenantFactoryImpl.java | 98 +++++++
.../factory/CloudObjectStorageFactory.java | 61 +++++
.../LegalComplianceChangeServiceImpl.java | 137 ++++++++++
.../reference/messagebus/IMessageFactory.java | 29 ++
.../model/RecordMetadataDocument.java | 48 ++++
.../reference/model/SchemaDocument.java | 82 ++++++
.../reference/model/TenantInfoDocument.java | 34 +++
.../persistence/MongoDdmsClient.java | 38 +++
.../repository/QueryRepositoryImpl.java | 95 +++++++
.../RecordsMetadataRepositoryImpl.java | 174 ++++++++++++
.../repository/SchemaRepositoryImpl.java | 94 +++++++
.../security/BasicAuthSecurityConfig.java | 35 +++
.../reference/security/WhoamiController.java | 42 +++
.../service/BatchServiceReferenceImpl.java | 58 ++++
.../reference/service/MessageBusImpl.java | 60 +++++
.../reference/util/MongoClientHandler.java | 80 ++++++
.../util/ServiceAccountJwtClientImpl.java | 32 +++
.../src/main/resources/application.properties | 27 ++
.../reference/CloudStorageImplTest.java | 129 +++++++++
.../context/ProviderSpringContextTest.java | 35 +++
.../src/test/resources/application.properties | 28 ++
skaffold.yaml | 12 +
37 files changed, 2331 insertions(+)
create mode 100644 provider/storage-reference/Dockerfile
create mode 100644 provider/storage-reference/docker-compose.yml
create mode 100644 provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml
create mode 100644 provider/storage-reference/pom.xml
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java
create mode 100644 provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java
create mode 100644 provider/storage-reference/src/main/resources/application.properties
create mode 100644 provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java
create mode 100644 provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java
create mode 100644 provider/storage-reference/src/test/resources/application.properties
create mode 100644 skaffold.yaml
diff --git a/pom.xml b/pom.xml
index 699b45e12..4449f5df5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,6 +118,7 @@
<module>provider/storage-azure</module>
<module>provider/storage-aws</module>
<module>provider/storage-ibm</module>
+ <module>provider/storage-reference</module>
</modules>
<repositories>
diff --git a/provider/storage-reference/Dockerfile b/provider/storage-reference/Dockerfile
new file mode 100644
index 000000000..ff9eb802f
--- /dev/null
+++ b/provider/storage-reference/Dockerfile
@@ -0,0 +1,5 @@
+FROM openjdk:8-slim
+WORKDIR /app
+COPY target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar storage-reference.jar
+# Run the web service on container startup.
+CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/storage-reference.jar
diff --git a/provider/storage-reference/docker-compose.yml b/provider/storage-reference/docker-compose.yml
new file mode 100644
index 000000000..213fdb3eb
--- /dev/null
+++ b/provider/storage-reference/docker-compose.yml
@@ -0,0 +1,12 @@
+version: "3"
+services:
+ os-storage-app:
+ build:
+ args:
+ JAR_FILE: target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar
+ context: ""
+ dockerfile: ../Dockerfile
+ image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference
+ ports:
+ - "8080:8080"
+
diff --git a/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml
new file mode 100644
index 000000000..9ff958c68
--- /dev/null
+++ b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml
@@ -0,0 +1,132 @@
+apiVersion: v1
+data:
+ AUTHORIZE_API: ${AUTHORIZE_API}
+ MONGO_DB_URL: ${MONGO_DB_URL}
+ MONGO_DB_USER: ${MONGO_DB_USER}
+ MONGO_DB_NAME: ${MONGO_DB_NAME}
+ REGION: ${REGION}
+ LEGALTAG_API: ${LEGALTAG_API}
+ org.opengroup.osdu.storage.disableAuth: "true"
+kind: ConfigMap
+metadata:
+ labels:
+ app: storage-reference
+ name: storage-config
+ namespace: default
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ generateName: storage-reference-anthos
+ labels:
+ app: storage-reference
+ name: storage-reference
+ namespace: default
+spec:
+ selector:
+ matchLabels:
+ app: storage-reference
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: storage-reference
+ spec:
+ containers:
+ - env:
+ - name: AUTHORIZE_API
+ valueFrom:
+ configMapKeyRef:
+ key: AUTHORIZE_API
+ name: storage-config
+ - name: MONGO_DB_URL
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_URL
+ name: storage-config
+ - name: MONGO_DB_USER
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_USER
+ name: storage-config
+ - name: MONGO_DB_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: storage-secret
+ key: mongo.db.password
+ - name: MONGO_DB_NAME
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_NAME
+ name: storage-config
+ - name: MB_RABBITMQ_URI
+ valueFrom:
+ secretKeyRef:
+ name: storage-secret
+ key: mb.rabbitmq.uri
+ - name: REGION
+ valueFrom:
+ configMapKeyRef:
+ key: REGION
+ name: storage-config
+ - name: org.opengroup.osdu.storage.disableAuth
+ valueFrom:
+ configMapKeyRef:
+ key: org.opengroup.osdu.storage.disableAuth
+ name: storage-config
+ - name: MINIO_URL
+ valueFrom:
+ secretKeyRef:
+ key: minio.enpoint_url
+ name: storage-secret
+ - name: MINIO_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ key: minio.access_key
+ name: storage-secret
+ - name: MINIO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: minio.secret_key
+ name: storage-secret
+ - name: MINIO_REGION
+ valueFrom:
+ secretKeyRef:
+ key: minio.region
+ name: storage-secret
+ - name: MINIO_BUCKET_RECORD_NAME
+ valueFrom:
+ secretKeyRef:
+ key: minio.bucket.record.name
+ name: storage-secret
+ image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference:9a1d20e-dirty
+ name: storage-reference
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-reference
+ namespace: default
+spec:
+ ports:
+ - protocol: TCP
+ port: 80
+ targetPort: 8080
+ selector:
+ app: storage-reference
+ type: LoadBalancer
+---
+apiVersion: v1
+data:
+ mongo.db.password: ${MONGO_DB_PASSWORD}
+ mb.rabbitmq.uri: ${MB_RABBITMQ_URI}
+ minio.enpoint_url: ${MINIO_URL}
+ minio.access_key: ${MINIO_ACCESS_KEY}
+ minio.secret_key: ${MINIO_SECRET_KEY}
+ minio.region: ${MINIO_REGION}
+ minio.bucket.record.name: ${MINIO_RECORD_BUCKET_NAME}
+kind: Secret
+metadata:
+ name: storage-secret
+ namespace: default
+type: Opaque
diff --git a/provider/storage-reference/pom.xml b/provider/storage-reference/pom.xml
new file mode 100644
index 000000000..88e8c2af9
--- /dev/null
+++ b/provider/storage-reference/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright © Microsoft Corporation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>os-storage</artifactId>
+ <groupId>org.opengroup.osdu</groupId>
+ <version>0.0.5-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>storage-reference</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>storage-core</artifactId>
+ <version>0.0.5-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.10.19</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-jose</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-resource-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.rabbitmq</groupId>
+ <artifactId>amqp-client</artifactId>
+ <version>5.7.3</version>
+ </dependency>
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ <version>7.1.4</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.storage.provider.reference.app.StorageReferenceApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java
new file mode 100644
index 000000000..210dfed55
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java
@@ -0,0 +1,247 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference;
+
+import static org.apache.commons.codec.binary.Base64.encodeBase64;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import io.minio.GetObjectArgs;
+import io.minio.MinioClient;
+import io.minio.PutObjectArgs;
+import io.minio.RemoveObjectArgs;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.annotation.PostConstruct;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
+import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.core.common.model.storage.TransferInfo;
+import org.opengroup.osdu.core.common.util.Crc32c;
+import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage;
+import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties;
+import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class CloudStorageImpl implements ICloudStorage {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CloudStorageImpl.class);
+
+ private final MinIoConfigProperties minIoConfigProperties;
+ private final CloudObjectStorageFactory factory;
+
+ private MinioClient minioClient;
+
+ public CloudStorageImpl(CloudObjectStorageFactory factory,
+ MinIoConfigProperties minIoConfigProperties) {
+ this.factory = factory;
+ this.minIoConfigProperties = minIoConfigProperties;
+ }
+
+ @PostConstruct
+ public void init() {
+ minioClient = factory.getClient();
+ }
+
+ @Override
+ public void write(RecordProcessing... recordsProcessing) {
+ Gson gson = new GsonBuilder().serializeNulls().create();
+ for (RecordProcessing rp : recordsProcessing) {
+ Map<String, String> headers = new HashMap<>();
+ headers.put("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE);
+ headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY");
+
+ String content = gson.toJson(rp.getRecordData());
+ byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+ String itemName = getItemName(rp.getRecordMetadata()).replace(":", "-");
+ try {
+ minioClient.putObject(
+ PutObjectArgs.builder()
+ .bucket(minIoConfigProperties.getMinIoBucketRecordName())
+ .object(itemName)
+ .stream(new ByteArrayInputStream(bytes), bytes.length, -1)
+ .headers(headers)
+ .build());
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ "Failed to write new record.", e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public Map<String, String> getHash(Collection<RecordMetadata> records) {
+ Gson gson = new Gson();
+ Map<String, String> hashes = new HashMap<>();
+ for (RecordMetadata rm : records) {
+ String jsonData = read(rm, rm.getLatestVersion(), false);
+ RecordData data = gson.fromJson(jsonData, RecordData.class);
+
+ String hash = getHash(data);
+ hashes.put(rm.getId(), hash);
+ }
+ return hashes;
+ }
+
+ @Override
+ public boolean isDuplicateRecord(TransferInfo transfer, Map<String, String> hashMap,
+ Map.Entry<RecordMetadata, RecordData> kv) {
+ RecordMetadata updatedRecordMetadata = kv.getKey();
+ RecordData recordData = kv.getValue();
+ String recordHash = hashMap.get(updatedRecordMetadata.getId());
+
+ String newHash = getHash(recordData);
+
+ if (newHash.equals(recordHash)) {
+ transfer.getSkippedRecords().add(updatedRecordMetadata.getId());
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private String getHash(RecordData data) {
+ Gson gson = new Gson();
+ Crc32c checksumGenerator = new Crc32c();
+
+ String newRecordStr = gson.toJson(data);
+ byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8);
+ checksumGenerator.update(bytes, 0, bytes.length);
+ bytes = checksumGenerator.getValueAsBytes();
+ String newHash = new String(encodeBase64(bytes));
+ return newHash;
+ }
+
+ private String getItemName(RecordMetadata record) {
+ return record.getVersionPath(record.getLatestVersion());
+ }
+
+ private String getItemName(RecordMetadata record, Long version) {
+ return record.getVersionPath(version);
+ }
+
+ @Override
+ public void delete(RecordMetadata record) {
+ String itemName = getItemName(record).replace(":", "-");
+ try {
+ minioClient.removeObject(
+ RemoveObjectArgs.builder()
+ .bucket(minIoConfigProperties.getMinIoBucketRecordName())
+ .object(itemName)
+ .build());
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete record.",
+ e.getMessage());
+ }
+ }
+
+ @Override
+ public void deleteVersion(RecordMetadata record, Long version) {
+ String itemName = getItemName(record, version).replace(":", "-");
+ try {
+ if (!record.hasVersion()) {
+ LOGGER.warn(String.format("Record %s does not have versions available", record.getId()));
+ }
+ minioClient.removeObject(
+ RemoveObjectArgs.builder()
+ .bucket(minIoConfigProperties.getMinIoBucketRecordName())
+ .object(itemName)
+ .build());
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete version.",
+ e.getMessage());
+ }
+ }
+
+ @Override
+ public boolean hasAccess(RecordMetadata... records) {
+ for (RecordMetadata record : records) {
+ if (!record.getStatus().equals(RecordState.active)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) {
+ String itemName = getItemName(record, version).replace(":", "-");
+ String msg = String
+ .format("Record with id '%s' does not exist, version: %s", record.getId(), version);
+ InputStream stream;
+ try {
+ stream = minioClient.getObject(
+ GetObjectArgs.builder()
+ .bucket(minIoConfigProperties.getMinIoBucketRecordName())
+ .object(itemName)
+ .build());
+ if (stream == null) {
+ LOGGER.warn(msg);
+ } else {
+ return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines()
+ .collect(Collectors.joining("\n"));
+ }
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.",
+ e.getMessage());
+ }
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg);
+ }
+
+ @Override
+ public Map<String, String> read(Map<String, String> objects) {
+ // key -> record id
+ // value -> record version path
+ Map<String, String> map = new HashMap<>();
+ for (Map.Entry<String, String> record : objects.entrySet()) {
+ String[] tokens = record.getValue().split("/");
+ String key = tokens[tokens.length - 2];
+ try {
+ InputStream stream = minioClient.getObject(
+ GetObjectArgs.builder()
+ .bucket(minIoConfigProperties.getMinIoBucketRecordName())
+ .object(record.getValue().replace(":", "-"))
+ .build());
+ if (stream != null) {
+ String result = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))
+ .lines()
+ .collect(Collectors.joining("\n"));
+ map.put(key, result);
+ }
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.",
+ e.getMessage());
+ }
+ }
+ return map;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java
new file mode 100644
index 000000000..590ab40cb
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.app;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+
+@SpringBootApplication(exclude = {MongoAutoConfiguration.class})
+@ComponentScan({"org.opengroup.osdu"})
+public class StorageReferenceApplication {
+
+ public static void main(String[] args) {
+
+ SpringApplication.run(StorageReferenceApplication.class, args);
+ }
+
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java
new file mode 100644
index 000000000..60eaab2f3
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.entitlements.Groups;
+import org.opengroup.osdu.storage.provider.reference.config.RedisConfigProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class GroupCache extends RedisCache<String, Groups> {
+
+ @Autowired
+ public GroupCache(RedisConfigProperties redisConfigProperties) {
+ super(redisConfigProperties.getGcpRedisHost(),
+ Integer.parseInt(redisConfigProperties.getGcpRedisPort()),
+ Integer.parseInt(redisConfigProperties.getGcpRedisExpTime()), String.class, Groups.class);
+ }
+}
\ No newline at end of file
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java
new file mode 100644
index 000000000..0388780bc
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.cache.MultiTenantCache;
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component("LegalTagCache")
+public class LegalTagCache implements ICache<String, String> {
+
+ @Autowired
+ private TenantInfo tenant;
+
+ private final MultiTenantCache<String> caches;
+
+ public LegalTagCache() {
+ this.caches = new MultiTenantCache<>(
+ new VmCache(60 * 60, 1000));
+ }
+
+ @Override
+ public void put(String key, String val) {
+ this.partitionCache().put(key, val);
+ }
+
+ @Override
+ public String get(String key) {
+ return this.partitionCache().get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ this.partitionCache().delete(key);
+ }
+
+ @Override
+ public void clearAll() {
+ this.partitionCache().clearAll();
+ }
+
+ private ICache<String, String> partitionCache() {
+ return this.caches.get(String.format("%s:legalTag", this.tenant));
+ }
+}
+
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java
new file mode 100644
index 000000000..e95694fdc
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCache extends VmCache<String, Schema> {
+
+ public SchemaCache() {
+ super(5 * 60, 1000);
+ }
+}
\ No newline at end of file
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java
new file mode 100644
index 000000000..d5000b775
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class MinIoConfigProperties {
+
+ private String minIoSignedUrlExpirationDays;
+ private String minIoEndpointUrl;
+ private String minIoAccessKey;
+ private String minIoSecretKey;
+ private String minIoRegion;
+ private String minIoPrefix;
+ private String minIoBucketRecordName;
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java
new file mode 100644
index 000000000..5438065ea
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class MongoDBConfigProperties {
+
+ private String mongoDbUrl;
+ private String mongoDbUser;
+ private String mongoDbPassword;
+ private String mongoDbName;
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java
new file mode 100644
index 000000000..f72efd6bc
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class RabbitMqConfigProperties {
+
+ private String mbRabbitMqUri;
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java
new file mode 100644
index 000000000..76455e490
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class RedisConfigProperties {
+
+ private String gcpRedisPort;
+ private String gcpRedisHost;
+ private String gcpRedisExpTime;
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java
new file mode 100644
index 000000000..0e5476784
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.di;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.PostConstruct;
+import org.opengroup.osdu.storage.provider.reference.config.RabbitMqConfigProperties;
+import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+@Lazy
+@Component
+public class RabbitMQFactoryImpl implements IMessageFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class);
+ private final RabbitMqConfigProperties rabbitMqConfigProperties;
+
+ private Channel channel;
+
+ @Autowired
+ public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) {
+ this.rabbitMqConfigProperties = rabbitMqConfigProperties;
+ }
+
+ @PostConstruct
+ private void init() {
+ ConnectionFactory factory = new ConnectionFactory();
+ try {
+ String mbRabbitMqUri = rabbitMqConfigProperties.getMbRabbitMqUri();
+ LOG.debug(String.format("RabbitMQ Uri = %s", mbRabbitMqUri));
+ factory.setUri(mbRabbitMqUri);
+ factory.setAutomaticRecoveryEnabled(true);
+ Connection conn = factory.newConnection();
+ this.channel = conn.createChannel();
+ LOG.debug("RabbitMQ Channel was created.");
+ for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) {
+ channel.queueDeclare(queue, true, false, false, null);
+ LOG.debug(String.format("Queue [ %s ] was declared.", queue));
+ }
+ } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void sendMessage(String msg) {
+ sendMessage(DEFAULT_QUEUE_NAME, msg);
+ }
+
+ @Override
+ public void sendMessage(String queueName, String msg) {
+ String queueNameWithPrefix = queueName;
+ try {
+ channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes());
+ LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix));
+ } catch (IOException e) {
+ LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix));
+ LOG.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java
new file mode 100644
index 000000000..4307d1ca9
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.di;
+
+import static java.util.Objects.isNull;
+
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.bson.Document;
+import org.bson.types.ObjectId;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
+ public static final String TENANT_INFO = "tenantinfo";
+ public static final String MAIN_DATABASE = "main";
+
+ @Autowired
+ private MongoDdmsClient mongoClient;
+
+ private Map<String, TenantInfo> tenants;
+
+ public boolean exists(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.containsKey(tenantName);
+ }
+
+ public TenantInfo getTenantInfo(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.get(tenantName);
+ }
+
+ public Collection<TenantInfo> listTenantInfo() {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.values();
+ }
+
+ public <V> ICache<String, V> createCache(String tenantName, String host, int port,
+ int expireTimeSeconds, Class<V> classOfV) {
+ return null;
+ }
+
+ public void flushCache() {
+ }
+
+ private void initTenants() {
+ this.tenants = new HashMap<>();
+ MongoCollection<Document> mongoCollection = mongoClient
+ .getMongoCollection(MAIN_DATABASE, TENANT_INFO);
+ FindIterable<Document> results = mongoCollection.find();
+ if (isNull(results) || isNull(results.first())) {
+ LOG.error(String.format("Collection \'%s\' is empty.", results));
+ }
+ for (Document document : results) {
+ TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class);
+ ObjectId id = (ObjectId) document.get("_id");
+ tenantInfo.setId((long) id.getCounter());
+ tenantInfo.setCrmAccountIds((ArrayList<String>) document.get("crmAccountID"));
+ this.tenants.put(tenantInfo.getName(), tenantInfo);
+ }
+ }
+}
+
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java
new file mode 100644
index 000000000..c4db1f818
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.factory;
+
+import io.minio.MinioClient;
+import javax.annotation.PostConstruct;
+import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+@Component
+@Lazy
+public class CloudObjectStorageFactory {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CloudObjectStorageFactory.class);
+ private final MinIoConfigProperties minIoConfigProperties;
+
+ private MinioClient minioClient;
+
+ @Autowired
+ public CloudObjectStorageFactory(MinIoConfigProperties minIoConfigProperties) {
+ this.minIoConfigProperties = minIoConfigProperties;
+ }
+
+ @PostConstruct
+ public void init() {
+ minioClient = MinioClient.builder()
+ .endpoint(minIoConfigProperties.getMinIoEndpointUrl())
+ .credentials(minIoConfigProperties.getMinIoAccessKey(),
+ minIoConfigProperties.getMinIoSecretKey())
+ .region(minIoConfigProperties.getMinIoRegion())
+ .build();
+ LOGGER.info("Minio client initialized");
+ }
+
+ public MinioClient getClient() {
+ return this.minioClient;
+ }
+
+ public void setMinioClient(MinioClient minioClient) {
+ this.minioClient = minioClient;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java
new file mode 100644
index 000000000..fe78a970c
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.jobs;
+
+import static java.util.Collections.singletonList;
+
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
+import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceChangeInfo;
+import org.opengroup.osdu.core.common.model.legal.jobs.ILegalComplianceChangeService;
+import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChanged;
+import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection;
+import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.storage.logging.StorageAuditLogger;
+import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
+import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
+import org.opengroup.osdu.storage.provider.reference.cache.LegalTagCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LegalComplianceChangeServiceImpl.class);
+
+ @Autowired
+ private IRecordsMetadataRepository recordsRepo;
+
+ @Autowired
+ private IMessageBus pubSubClient;
+
+ @Autowired
+ private StorageAuditLogger auditLogger;
+
+ @Autowired
+ private LegalTagCache legalTagCache;
+
+ @Override
+ public Map<String, LegalCompliance> updateComplianceOnRecords(
+ LegalTagChangedCollection legalTagsChanged,
+ DpsHeaders headers) {
+ Map<String, LegalCompliance> output = new HashMap<>();
+
+ for (LegalTagChanged lt : legalTagsChanged.getStatusChangedTags()) {
+
+ ComplianceChangeInfo complianceChangeInfo = this.getComplianceChangeInfo(lt);
+ if (complianceChangeInfo == null) {
+ continue;
+ }
+
+ String cursor = null;
+ do {
+ //TODO replace with the new method queryByLegal
+ AbstractMap.SimpleEntry<String, List<RecordMetadata>> results = this.recordsRepo
+ .queryByLegalTagName(lt.getChangedTagName(), 500, cursor);
+ cursor = results.getKey();
+
+ if (results.getValue() != null && !results.getValue().isEmpty()) {
+ List<RecordMetadata> recordsMetadata = results.getValue();
+ PubSubInfo[] pubSubInfos = this
+ .updateComplianceStatus(complianceChangeInfo, recordsMetadata, output);
+ this.recordsRepo.createOrUpdate(recordsMetadata);
+ StringBuilder recordsId = new StringBuilder();
+ for (RecordMetadata recordMetadata : recordsMetadata) {
+ recordsId.append(", ").append(recordMetadata.getId());
+ }
+ this.auditLogger.updateRecordsComplianceStateSuccess(
+ singletonList("[" + recordsId.substring(2) + "]"));
+
+ this.pubSubClient.publishMessage(headers, pubSubInfos);
+ }
+ } while (cursor != null);
+ }
+
+ return output;
+ }
+
+ private PubSubInfo[] updateComplianceStatus(ComplianceChangeInfo complianceChangeInfo,
+ List<RecordMetadata> recordMetadata, Map<String, LegalCompliance> output) {
+
+ PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()];
+
+ int i = 0;
+ for (RecordMetadata rm : recordMetadata) {
+ rm.getLegal().setStatus(complianceChangeInfo.getNewState());
+ rm.setStatus(complianceChangeInfo.getNewRecordState());
+ pubsubInfo[i] = new PubSubInfo(rm.getId(), rm.getKind(),
+ complianceChangeInfo.getPubSubEvent());
+ output.put(rm.getId(), complianceChangeInfo.getNewState());
+ i++;
+ }
+
+ return pubsubInfo;
+ }
+
+ private ComplianceChangeInfo getComplianceChangeInfo(LegalTagChanged lt) {
+ ComplianceChangeInfo output = null;
+
+ if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) {
+ output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create,
+ RecordState.active);
+ } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) {
+ this.legalTagCache.delete(lt.getChangedTagName());
+ output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete,
+ RecordState.deleted);
+ } else {
+ LOG.warn(String.format("Unknown LegalTag compliance status received %s %s",
+ lt.getChangedTagStatus(), lt.getChangedTagName()));
+ }
+
+ return output;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java
new file mode 100644
index 000000000..4db9ec80f
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.messagebus;
+
+public interface IMessageFactory {
+
+ String DEFAULT_QUEUE_NAME = "records";
+ String LEGAL_QUEUE_NAME = "legal";
+ String INDEXER_QUEUE_NAME = "indexer";
+
+ void sendMessage(String msg);
+
+ void sendMessage(String queueName, String msg);
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java
new file mode 100644
index 000000000..0038bb9ec
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.legal.Legal;
+import org.opengroup.osdu.core.common.model.storage.RecordAncestry;
+import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Document(collection = "StorageRecord")
+public class RecordMetadataDocument {
+
+ private String id;
+ private String kind;
+ private Acl acl;
+ private Legal legal;
+ private RecordAncestry ancestry;
+ private List<String> gcsVersionPaths = new ArrayList();
+ private RecordState status;
+ private String user;
+ private Long createTime;
+ private String modifyUser;
+ private Long modifyTime;
+}
\ No newline at end of file
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java
new file mode 100644
index 000000000..65bfe11a2
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.model;
+
+import java.util.Map;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Document(collection = "StorageSchema")
+public class SchemaDocument {
+
+ private String kind;
+ private String rev;
+ private String user;
+ private SchemaItem[] schema;
+ private Map<String, Object> extension;
+
+
+ public SchemaDocument(Schema schema, String user) {
+ this.setKind(schema.getKind());
+ this.setExtension(schema.getExt());
+ this.setSchema(schema.getSchema());
+ this.setUser(user);
+ }
+
+ public String getKind() {
+ return kind;
+ }
+
+ public void setKind(String kind) {
+ this.kind = kind;
+ }
+
+ public String getRev() {
+ return rev;
+ }
+
+ public void setRev(String rev) {
+ this.rev = rev;
+ }
+
+ public Map<String, Object> getExtension() {
+ return extension;
+ }
+
+ public void setExtension(Map<String, Object> extension) {
+ this.extension = extension;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public SchemaItem[] getSchema() {
+ return schema;
+ }
+
+ public void setSchema(SchemaItem[] schemaItems) {
+ this.schema = schemaItems;
+ }
+
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java
new file mode 100644
index 000000000..ae2ea414f
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.model;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Document(collection = "TenantInfo")
+public class TenantInfoDocument {
+
+ private String id;
+ private List<String> groups;
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java
new file mode 100644
index 000000000..accf7016d
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.persistence;
+
+import com.mongodb.client.MongoCollection;
+import org.bson.Document;
+import org.opengroup.osdu.storage.provider.reference.util.MongoClientHandler;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoDdmsClient {
+
+ private MongoClientHandler mongoClientHandler;
+
+ public MongoDdmsClient(MongoClientHandler mongoClientHandler) {
+ this.mongoClientHandler = mongoClientHandler;
+ }
+
+ public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) {
+ return mongoClientHandler.getMongoClient().getDatabase(dbName)
+ .getCollection(collectionName);
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java
new file mode 100644
index 000000000..8c165109b
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.repository;
+
+import static com.mongodb.client.model.Filters.eq;
+import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.RECORD_STORAGE;
+import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE;
+import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_STORAGE;
+
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
+import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
+import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
+import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository;
+import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class QueryRepositoryImpl implements IQueryRepository {
+ private MongoDdmsClient mongoDdmsClient;
+
+ @Autowired
+ public QueryRepositoryImpl(MongoDdmsClient mongoDdmsClient) {
+ this.mongoDdmsClient = mongoDdmsClient;
+ }
+
+ @Override
+ public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
+ int numRecords = PAGE_SIZE;
+ if (limit != null) {
+ numRecords = limit > 0 ? limit : PAGE_SIZE;
+ }
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE);
+ FindIterable<Document> results = mongoCollection.find()
+ .limit(numRecords);
+ List<String> kinds = new ArrayList<>();
+ for (Document document : results) {
+ kinds.add(document.get("kind").toString());
+ }
+ return new DatastoreQueryResult(cursor, kinds);
+ }
+
+ @Override
+ public DatastoreQueryResult getAllRecordIdsFromKind(
+ String kind, Integer limit, String cursor) {
+ boolean paginated = false;
+
+ int numRecords = PAGE_SIZE;
+ if (Objects.nonNull(limit)) {
+ numRecords = limit > 0 ? limit : PAGE_SIZE;
+ paginated = true;
+ }
+
+ if (cursor != null && !cursor.isEmpty()) {
+ paginated = true;
+ }
+
+ DatastoreQueryResult dqr = new DatastoreQueryResult();
+ List<String> ids = new ArrayList();
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, RECORD_STORAGE);
+ FindIterable<Document> results = mongoCollection.find(eq("kind", kind))
+ .limit(numRecords);
+ for (Document document : results) {
+ ids.add(document.get("id").toString());
+ }
+ dqr.setResults(ids);
+ dqr.setCursor(cursor);
+ return dqr;
+ }
+
+}
+
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java
new file mode 100644
index 000000000..b69f712c2
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.repository;
+
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.util.JSON.serialize;
+import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE;
+
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.UpdateOptions;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
+import org.opengroup.osdu.storage.provider.reference.model.RecordMetadataDocument;
+import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository<String> {
+
+ public static final String STORAGE_RECORD = "StorageRecord";
+ private final MongoDdmsClient mongoDdmsClient;
+
+ @Autowired
+ public RecordsMetadataRepositoryImpl(MongoDdmsClient mongoDdmsClient) {
+ this.mongoDdmsClient = mongoDdmsClient;
+ }
+
+ @Override
+ public List<RecordMetadata> createOrUpdate(List<RecordMetadata> recordsMetadata) {
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD);
+ if (Objects.nonNull(recordsMetadata)) {
+ for (RecordMetadata recordMetadata : recordsMetadata) {
+ RecordMetadataDocument recordMetadataDocument = convertToRecordMetadataDocument(
+ recordMetadata);
+ mongoCollection.replaceOne(eq("id", recordMetadataDocument.getId()),
+ Document.parse(new Gson().toJson(recordMetadataDocument)),
+ (new UpdateOptions()).upsert(true));
+ }
+ }
+ return recordsMetadata;
+ }
+
+ @Override
+ public void delete(String id) {
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD);
+ mongoCollection.deleteOne(eq("id", id));
+ }
+
+ @Override
+ public RecordMetadata get(String id) {
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD);
+ Document doc = mongoCollection.find(eq("id", id)).first();
+ if (Objects.isNull(doc)) {
+ return null;
+ }
+ RecordMetadataDocument recordMetadataDocument = new Gson()
+ .fromJson(serialize(doc), RecordMetadataDocument.class);
+ return convertToRecordMetadata(recordMetadataDocument);
+ }
+
+ @Override
+ public Map<String, RecordMetadata> get(List<String> ids) {
+ Map<String, RecordMetadata> output = new HashMap<>();
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD);
+ for (String id : ids) {
+ Document document = mongoCollection.find(eq("id", id)).first();
+ RecordMetadataDocument recordMetadataDocument = null;
+ if (Objects.nonNull(document)) {
+ recordMetadataDocument = new Gson()
+ .fromJson(serialize(document), RecordMetadataDocument.class);
+ }
+ RecordMetadata rmd = convertToRecordMetadata(recordMetadataDocument);
+ if (Objects.isNull(rmd)) {
+ continue;
+ }
+ output.put(id, rmd);
+ }
+ return output;
+ }
+
+ @Override
+ public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName(
+ String legalTagName, int limit, String cursor) {
+ MongoCollection<Document> mongoCollection = mongoDdmsClient
+ .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD);
+ List<RecordMetadata> outputRecords = new ArrayList<>();
+ FindIterable<Document> results = mongoCollection.find().skip(limit * (limit - 1)).limit(limit);
+ for (Document document : results) {
+ RecordMetadataDocument recordMetadataDocument = new Gson()
+ .fromJson(serialize(document), RecordMetadataDocument.class);
+ if (Objects.nonNull(recordMetadataDocument)) {
+ if (recordMetadataDocument.getLegal().getLegaltags().contains(legalTagName)) {
+ RecordMetadata recordMetadata = convertToRecordMetadata(recordMetadataDocument);
+ outputRecords.add(recordMetadata);
+ }
+ }
+ }
+ return new AbstractMap.SimpleEntry<>(cursor, outputRecords);
+ }
+
+ @Override
+ public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegal(String legalTagName,
+ LegalCompliance status, int limit) {
+ return null;
+ }
+
+ private RecordMetadataDocument convertToRecordMetadataDocument(RecordMetadata recordMetadata) {
+ RecordMetadataDocument recordMetadataDocument = new RecordMetadataDocument();
+ recordMetadataDocument.setId(recordMetadata.getId());
+ recordMetadataDocument.setAcl(recordMetadata.getAcl());
+ recordMetadataDocument.setAncestry(recordMetadata.getAncestry());
+ recordMetadataDocument.setCreateTime(recordMetadata.getCreateTime());
+ recordMetadataDocument.setModifyTime(recordMetadata.getModifyTime());
+ recordMetadataDocument.setGcsVersionPaths(recordMetadata.getGcsVersionPaths());
+ recordMetadataDocument.setKind(recordMetadata.getKind());
+ recordMetadataDocument.setLegal(recordMetadata.getLegal());
+ recordMetadataDocument.setModifyUser(recordMetadata.getModifyUser());
+ recordMetadataDocument.setStatus(recordMetadata.getStatus());
+ recordMetadataDocument.setUser(recordMetadata.getUser());
+
+ return recordMetadataDocument;
+ }
+
+ private RecordMetadata convertToRecordMetadata(RecordMetadataDocument recordMetadataDocument) {
+ if (Objects.isNull(recordMetadataDocument)) {
+ return null;
+ }
+ RecordMetadata recordMetadata = new RecordMetadata();
+ recordMetadata.setId(recordMetadataDocument.getId());
+ recordMetadata.setAcl(recordMetadataDocument.getAcl());
+ recordMetadata.setAncestry(recordMetadataDocument.getAncestry());
+ recordMetadata.setCreateTime(recordMetadataDocument.getCreateTime());
+ recordMetadata.setModifyTime(recordMetadataDocument.getModifyTime());
+ recordMetadata.setGcsVersionPaths(recordMetadataDocument.getGcsVersionPaths());
+ recordMetadata.setKind(recordMetadataDocument.getKind());
+ recordMetadata.setLegal(recordMetadataDocument.getLegal());
+ recordMetadata.setModifyUser(recordMetadataDocument.getModifyUser());
+ recordMetadata.setStatus(recordMetadataDocument.getStatus());
+ recordMetadata.setUser(recordMetadataDocument.getUser());
+
+ return recordMetadata;
+ }
+}
+
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java
new file mode 100644
index 000000000..c8ae6fd72
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.repository;
+
+import static com.mongodb.client.model.Filters.eq;
+
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Objects;
+import org.apache.http.HttpStatus;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository;
+import org.opengroup.osdu.storage.provider.reference.model.SchemaDocument;
+import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class SchemaRepositoryImpl implements ISchemaRepository {
+
+ private static final Logger logger = LoggerFactory.getLogger(SchemaRepositoryImpl.class);
+ public static final String SCHEMA_STORAGE = "SchemaStorage";
+ public static final String RECORD_STORAGE = "StorageRecord";
+ public static final String SCHEMA_DATABASE = "schema";
+
+ private MongoDdmsClient mongoClient;
+
+ @Autowired
+ public SchemaRepositoryImpl(MongoDdmsClient mongoClient) {
+ this.mongoClient = mongoClient;
+ }
+
+ @Override
+ public void add(Schema schema, String user) {
+ MongoCollection collection = this.mongoClient
+ .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE);
+ String kind = schema.getKind();
+ FindIterable<Document> results = collection.find(eq("kind", kind));
+ if (Objects.nonNull(results) && Objects.nonNull(results.first())) {
+ throw new IllegalArgumentException("Schema " + kind + " already exist. Can't create again.");
+ }
+ SchemaDocument schemaDocument = new SchemaDocument(schema, user);
+ collection.insertOne(Document.parse(new Gson().toJson(schemaDocument)));
+ }
+
+ @Override
+ public Schema get(String kind) {
+ MongoCollection collection = this.mongoClient
+ .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE);
+ Document record = (Document) collection.find(eq("kind", kind)).first();
+ if (Objects.isNull(record)) {
+ throw new AppException(
+ HttpStatus.SC_NOT_FOUND, "Not found",
+ String.format("Schema with id %s does not exist.", kind));
+ }
+ SchemaDocument schemaDocument = new Gson().fromJson(record.toJson(), SchemaDocument.class);
+ return convertToSchemaEntity(schemaDocument);
+ }
+
+ @Override
+ public void delete(String kind) {
+ MongoCollection collection = this.mongoClient
+ .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE);
+ collection.deleteOne(eq("kind", kind));
+ }
+
+ private Schema convertToSchemaEntity(SchemaDocument schemaDocument) {
+ Schema schema = new Schema();
+ schema.setKind(schemaDocument.getKind());
+ schema.setExt(schemaDocument.getExtension());
+ schema.setSchema(schemaDocument.getSchema());
+ return schema;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java
new file mode 100644
index 000000000..ff2a17f6e
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.security;
+
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http
+ .httpBasic().disable()
+ .csrf().disable();
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java
new file mode 100644
index 000000000..76eba1e23
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.security;
+
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+public class WhoamiController {
+
+ @RequestMapping(value = {"/", "/whoami"})
+ @ResponseBody
+ public String whoami() {
+ final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+
+ String userName = auth.getName();
+ String roles = String.valueOf(auth.getAuthorities());
+ String details = String.valueOf(auth.getPrincipal());
+
+ return "user: " + userName + "<BR>" +
+ "roles: " + roles + "<BR>" +
+ "details: " + details;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java
new file mode 100644
index 000000000..7d073db5b
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.service;
+
+import static java.util.Collections.singletonList;
+
+import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
+import org.opengroup.osdu.storage.logging.StorageAuditLogger;
+import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
+import org.opengroup.osdu.storage.service.BatchServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class BatchServiceReferenceImpl extends BatchServiceImpl {
+
+ private final StorageAuditLogger auditLogger;
+ private final IQueryRepository queryRepository;
+
+ @Autowired
+ public BatchServiceReferenceImpl(StorageAuditLogger auditLogger,
+ IQueryRepository queryRepository) {
+ this.auditLogger = auditLogger;
+ this.queryRepository = queryRepository;
+ }
+
+ @Override
+ public DatastoreQueryResult getAllKinds(String cursor, Integer limit) {
+ DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor);
+ this.auditLogger.readAllKindsSuccess(result.getResults());
+ return result;
+ }
+
+ @Override
+ public DatastoreQueryResult getAllRecords(String cursor, String kind, Integer limit) {
+ DatastoreQueryResult result = this.queryRepository.getAllRecordIdsFromKind(kind, limit, cursor);
+ if (!result.getResults().isEmpty()) {
+ this.auditLogger.readAllRecordsOfGivenKindSuccess(singletonList(kind));
+ }
+ return result;
+ }
+
+}
\ No newline at end of file
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java
new file mode 100644
index 000000000..409e01348
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.service;
+
+import com.google.gson.Gson;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
+import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
+import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MessageBusImpl implements IMessageBus {
+
+ private final IMessageFactory messageQueue;
+
+ @Autowired
+ public MessageBusImpl(IMessageFactory messageQueue) {
+ this.messageQueue = messageQueue;
+ }
+
+ public void publishMessage(DpsHeaders headers, PubSubInfo... messages) {
+ final int BATCH_SIZE = 50;
+ Map<String, String> message = new HashMap<>();
+ Gson gson = new Gson();
+
+ for (int i = 0; i < messages.length; i += BATCH_SIZE) {
+ PubSubInfo[] batch = Arrays
+ .copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE));
+
+ String json = gson.toJson(batch);
+ message.put("data", json);
+ message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ message.put(DpsHeaders.AUTHORIZATION, headers.getAuthorization());
+ messageQueue.sendMessage(gson.toJson(message));
+ }
+ }
+}
+
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java
new file mode 100644
index 000000000..7bbb0a9a9
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.util;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties;
+import org.opengroup.osdu.storage.provider.reference.config.MongoDBConfigProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoClientHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MongoClientHandler.class);
+ private static final String MONGO_PREFIX = "mongodb://";
+ private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000";
+
+ private com.mongodb.client.MongoClient mongoClient = null;
+ private MongoDBConfigProperties mongoDBConfigProperties;
+
+ private MongoClient getOrInitMongoClient() throws RuntimeException {
+ if (mongoClient != null) {
+ return mongoClient;
+ }
+
+ final String connectionString = String.format("%s%s:%s@%s/?%s",
+ MONGO_PREFIX,
+ mongoDBConfigProperties.getMongoDbUser(),
+ mongoDBConfigProperties.getMongoDbPassword(),
+ mongoDBConfigProperties.getMongoDbUrl(),
+ MONGO_OPTIONS);
+ ConnectionString connString = new ConnectionString(connectionString);
+ MongoClientSettings settings = MongoClientSettings.builder()
+ .applyConnectionString(connString)
+ .retryWrites(true)
+ .build();
+ try {
+ mongoClient = MongoClients.create(settings);
+ } catch (Exception ex) {
+ LOG.error("Error connecting MongoDB", ex);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB",
+ ex.getMessage(), ex);
+ }
+ return mongoClient;
+ }
+
+ public MongoClient getMongoClient() {
+ if (mongoClient == null) {
+ getOrInitMongoClient();
+ }
+ return mongoClient;
+ }
+
+ @Autowired
+ public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) {
+ this.mongoDBConfigProperties = mongoDBConfigProperties;
+ }
+}
diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java
new file mode 100644
index 000000000..07adc6bcb
--- /dev/null
+++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.util;
+
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+
+ @Override
+ public String getIdToken(String tenantName) {
+ return "dont-have-one";
+ }
+}
diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties
new file mode 100644
index 000000000..1e7c8dc1b
--- /dev/null
+++ b/provider/storage-reference/src/main/resources/application.properties
@@ -0,0 +1,27 @@
+LOG_PREFIX=storage
+server.servlet.contextPath=/api/storage/v2/
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1
+LEGALTAG_API=https://os-legal-ibm/api/legal/v1
+
+mongo-db-url=localhost:27017
+mongo-db-user=
+mongo-db-password=
+
+#amqp://guest:guest@127.0.0.1:5672/%2F by default
+mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672/%2F
+
+minio-endpoint-url=http://127.0.0.1:9000
+minio-access-key=
+minio-secret-key=
+minio-region=
+minio-prefix=local-dev
+minio-bucket-record-name=record-bucket
+
+gcp-redis-host=localhost
+gcp-redis-port=6379
+gcp-redis-exp-time=10
\ No newline at end of file
diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java
new file mode 100644
index 000000000..f4fb1e2f3
--- /dev/null
+++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import io.minio.MinioClient;
+import java.util.HashMap;
+import java.util.Map;
+import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.storage.Record;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
+import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties;
+import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory;
+
+@RunWith(MockitoJUnitRunner.class)
+public class CloudStorageImplTest extends TestCase {
+
+ private RecordProcessing[] recordProcessingArray = new RecordProcessing[1];
+ private RecordProcessing recordProcessing;
+ private RecordMetadata recordMetadata;
+ private String bucketName = "osdu-sample-osdu-file";
+
+ @Mock
+ private CloudObjectStorageFactory factory;
+
+ @Mock
+ private MinioClient minioClient;
+
+ @Mock
+ private MinIoConfigProperties minIoConfigProperties;
+
+ @InjectMocks
+ private CloudStorageImpl cloudStorage;
+
+ @Before
+ public void setup() {
+ recordMetadata = new RecordMetadata();
+ recordMetadata.setKind("test-record-id");
+ recordMetadata.setId("test-record-id");
+ recordMetadata.addGcsPath(1);
+ recordMetadata.addGcsPath(2);
+
+ recordProcessing = new RecordProcessing();
+ recordProcessing.setRecordMetadata(recordMetadata);
+
+ Record record = new Record();
+ record.setId("test-record-id");
+ Map<String, Object> data = new HashMap<>();
+ data.put("test-data", new Object());
+ record.setData(data);
+
+ RecordData recordData = new RecordData(record);
+ recordProcessing.setRecordData(recordData);
+ recordProcessingArray[0] = recordProcessing;
+ }
+
+ @Test
+ public void write_test() throws Exception {
+ when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName);
+
+ cloudStorage.write(recordProcessingArray);
+
+ verify(minioClient, times(1)).putObject(any());
+ }
+
+ @Test
+ public void delete_test() throws Exception {
+ when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName);
+
+ cloudStorage.delete(recordMetadata);
+
+ verify(minioClient, times(1)).removeObject(any());
+ }
+
+ @Test
+ public void deleteVersion_test() throws Exception {
+ when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName);
+
+ cloudStorage.deleteVersion(recordMetadata, 1L);
+
+ verify(minioClient, times(1)).removeObject(any());
+ }
+
+ @Test(expected = AppException.class)
+ public void read_with_version_test() throws Exception {
+ cloudStorage.read(recordMetadata, 1L, false);
+
+ verify(minioClient, times(1)).getObject(any());
+ }
+
+ @Test
+ public void read_test() throws Exception {
+ when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName);
+
+ Map<String, String> map = new HashMap<>();
+ map.put("common:welldb:1", "opendes:ds:mytest1:1.0.0/common:welldb:123456/1603618609515093");
+ cloudStorage.read(map);
+
+ verify(minioClient, times(1)).getObject(any());
+ }
+}
\ No newline at end of file
diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java
new file mode 100644
index 000000000..5fd9839ce
--- /dev/null
+++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.storage.provider.reference.context;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class ProviderSpringContextTest {
+
+ @Test
+ public void load_spring_context() {
+
+ }
+}
diff --git a/provider/storage-reference/src/test/resources/application.properties b/provider/storage-reference/src/test/resources/application.properties
new file mode 100644
index 000000000..7e2bae188
--- /dev/null
+++ b/provider/storage-reference/src/test/resources/application.properties
@@ -0,0 +1,28 @@
+LOG_PREFIX=storage
+
+server.servlet.contextPath=/api/storage/v2/
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1
+LEGALTAG_API=https://os-legal-ibm/api/legal/v1
+
+mongo.db.url=localhost:27017
+mongo.db.user=admin
+mongo.db.password=admin
+
+#amqp://guest:guest@127.0.0.1:5672/%2F by default
+mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F
+
+minio.endpoint.url=http://127.0.0.1:9000
+minio.access.key=adminadmin
+minio.secret.key=adminadmin
+minio.region=admin
+minio.prefix=local-dev
+minio.bucket.record.name=record-bucket
+
+gcp.redis.host=localhost
+gcp.redis.port=6379
+gcp.redis.exp.time=10
\ No newline at end of file
diff --git a/skaffold.yaml b/skaffold.yaml
new file mode 100644
index 000000000..4e5375a80
--- /dev/null
+++ b/skaffold.yaml
@@ -0,0 +1,12 @@
+apiVersion: skaffold/v2beta4
+kind: Config
+metadata:
+ name: storage-reference
+build:
+ artifacts:
+ - image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference
+ context: ./provider/storage-reference
+deploy:
+ kubectl:
+ manifests:
+ - ./provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml
\ No newline at end of file
--
GitLab