diff --git a/pom.xml b/pom.xml index 699b45e1222dc5ed987aa428e078e511a6991902..4449f5df5ec9ea64ce012365a205da526b04d087 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,7 @@ <module>provider/storage-azure</module> <module>provider/storage-aws</module> <module>provider/storage-ibm</module> + <module>provider/storage-reference</module> </modules> <repositories> diff --git a/provider/storage-reference/Dockerfile b/provider/storage-reference/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ff9eb802f72d671c0db8af8919892e8023485d3c --- /dev/null +++ b/provider/storage-reference/Dockerfile @@ -0,0 +1,5 @@ +FROM openjdk:8-slim +WORKDIR /app +COPY target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar storage-reference.jar +# Run the web service on container startup. +CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/storage-reference.jar diff --git a/provider/storage-reference/docker-compose.yml b/provider/storage-reference/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..213fdb3eb509d3a3013adcf44c046772907127c4 --- /dev/null +++ b/provider/storage-reference/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3" +services: + os-storage-app: + build: + args: + JAR_FILE: target/storage-reference-0.0.5-SNAPSHOT-spring-boot.jar + context: "" + dockerfile: ../Dockerfile + image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference + ports: + - "8080:8080" + diff --git a/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..9ff958c68628df1f17886ef3dee695531b8da5f1 --- /dev/null +++ b/provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml @@ -0,0 +1,132 @@ +apiVersion: v1 +data: + AUTHORIZE_API: ${AUTHORIZE_API} + MONGO_DB_URL: ${MONGO_DB_URL} + MONGO_DB_USER: ${MONGO_DB_USER} + MONGO_DB_NAME: ${MONGO_DB_NAME} + REGION: ${REGION} + LEGALTAG_API: ${LEGALTAG_API} + org.opengroup.osdu.storage.disableAuth: "true" +kind: ConfigMap +metadata: + labels: + app: storage-reference + name: storage-config + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + generateName: storage-reference-anthos + labels: + app: storage-reference + name: storage-reference + namespace: default +spec: + selector: + matchLabels: + app: storage-reference + replicas: 1 + template: + metadata: + labels: + app: storage-reference + spec: + containers: + - env: + - name: AUTHORIZE_API + valueFrom: + configMapKeyRef: + key: AUTHORIZE_API + name: storage-config + - name: MONGO_DB_URL + valueFrom: + configMapKeyRef: + key: MONGO_DB_URL + name: storage-config + - name: MONGO_DB_USER + valueFrom: + configMapKeyRef: + key: MONGO_DB_USER + name: storage-config + - name: MONGO_DB_PASSWORD + valueFrom: + secretKeyRef: + name: storage-secret + key: mongo.db.password + - name: MONGO_DB_NAME + valueFrom: + configMapKeyRef: + key: MONGO_DB_NAME + name: storage-config + - name: MB_RABBITMQ_URI + valueFrom: + secretKeyRef: + name: storage-secret + key: mb.rabbitmq.uri + - name: REGION + valueFrom: + configMapKeyRef: + key: REGION + name: storage-config + - name: org.opengroup.osdu.storage.disableAuth + valueFrom: + configMapKeyRef: + key: org.opengroup.osdu.storage.disableAuth + name: storage-config + - name: MINIO_URL + valueFrom: + secretKeyRef: + key: minio.enpoint_url + name: storage-secret + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: minio.access_key + name: storage-secret + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: minio.secret_key + name: storage-secret + - name: MINIO_REGION + valueFrom: + secretKeyRef: + key: minio.region + name: storage-secret + - name: MINIO_BUCKET_RECORD_NAME + valueFrom: + secretKeyRef: + key: minio.bucket.record.name + name: storage-secret + image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference:9a1d20e-dirty + name: storage-reference +--- +apiVersion: v1 +kind: Service +metadata: + name: storage-reference + namespace: default +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + selector: + app: storage-reference + type: LoadBalancer +--- +apiVersion: v1 +data: + mongo.db.password: ${MONGO_DB_PASSWORD} + mb.rabbitmq.uri: ${MB_RABBITMQ_URI} + minio.enpoint_url: ${MINIO_URL} + minio.access_key: ${MINIO_ACCESS_KEY} + minio.secret_key: ${MINIO_SECRET_KEY} + minio.region: ${MINIO_REGION} + minio.bucket.record.name: ${MINIO_RECORD_BUCKET_NAME} +kind: Secret +metadata: + name: storage-secret + namespace: default +type: Opaque diff --git a/provider/storage-reference/pom.xml b/provider/storage-reference/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..88e8c2af9b1e61071179bcc4ddeaba938f786169 --- /dev/null +++ b/provider/storage-reference/pom.xml @@ -0,0 +1,116 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Copyright © Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>os-storage</artifactId> + <groupId>org.opengroup.osdu</groupId> + <version>0.0.5-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>storage-reference</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>os-core-common</artifactId> + </dependency> + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>storage-core</artifactId> + <version>0.0.5-SNAPSHOT</version> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>1.10.19</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>2.0.2</version> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-mongodb</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-oauth2-jose</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-oauth2-resource-server</artifactId> + </dependency> + <dependency> + <groupId>com.rabbitmq</groupId> + <artifactId>amqp-client</artifactId> + <version>5.7.3</version> + </dependency> + <dependency> + <groupId>io.minio</groupId> + <artifactId>minio</artifactId> + <version>7.1.4</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + <configuration> + <classifier>spring-boot</classifier> + <mainClass> + org.opengroup.osdu.storage.provider.reference.app.StorageReferenceApplication + </mainClass> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..210dfed557c5cb894199f0224bb236d1ba70590a --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImpl.java @@ -0,0 +1,247 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference; + +import static org.apache.commons.codec.binary.Base64.encodeBase64; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.RecordData; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordProcessing; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.opengroup.osdu.core.common.model.storage.TransferInfo; +import org.opengroup.osdu.core.common.util.Crc32c; +import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; +import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Repository; + +@Repository +public class CloudStorageImpl implements ICloudStorage { + + private static final Logger LOGGER = LoggerFactory.getLogger(CloudStorageImpl.class); + + private final MinIoConfigProperties minIoConfigProperties; + private final CloudObjectStorageFactory factory; + + private MinioClient minioClient; + + public CloudStorageImpl(CloudObjectStorageFactory factory, + MinIoConfigProperties minIoConfigProperties) { + this.factory = factory; + this.minIoConfigProperties = minIoConfigProperties; + } + + @PostConstruct + public void init() { + minioClient = factory.getClient(); + } + + @Override + public void write(RecordProcessing... recordsProcessing) { + Gson gson = new GsonBuilder().serializeNulls().create(); + for (RecordProcessing rp : recordsProcessing) { + Map<String, String> headers = new HashMap<>(); + headers.put("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE); + headers.put("X-Amz-Storage-Class", "REDUCED_REDUNDANCY"); + + String content = gson.toJson(rp.getRecordData()); + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + String itemName = getItemName(rp.getRecordMetadata()).replace(":", "-"); + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .stream(new ByteArrayInputStream(bytes), bytes.length, -1) + .headers(headers) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, + "Failed to write new record.", e.getMessage()); + } + } + } + + @Override + public Map<String, String> getHash(Collection<RecordMetadata> records) { + Gson gson = new Gson(); + Map<String, String> hashes = new HashMap<>(); + for (RecordMetadata rm : records) { + String jsonData = read(rm, rm.getLatestVersion(), false); + RecordData data = gson.fromJson(jsonData, RecordData.class); + + String hash = getHash(data); + hashes.put(rm.getId(), hash); + } + return hashes; + } + + @Override + public boolean isDuplicateRecord(TransferInfo transfer, Map<String, String> hashMap, + Map.Entry<RecordMetadata, RecordData> kv) { + RecordMetadata updatedRecordMetadata = kv.getKey(); + RecordData recordData = kv.getValue(); + String recordHash = hashMap.get(updatedRecordMetadata.getId()); + + String newHash = getHash(recordData); + + if (newHash.equals(recordHash)) { + transfer.getSkippedRecords().add(updatedRecordMetadata.getId()); + return true; + } else { + return false; + } + } + + private String getHash(RecordData data) { + Gson gson = new Gson(); + Crc32c checksumGenerator = new Crc32c(); + + String newRecordStr = gson.toJson(data); + byte[] bytes = newRecordStr.getBytes(StandardCharsets.UTF_8); + checksumGenerator.update(bytes, 0, bytes.length); + bytes = checksumGenerator.getValueAsBytes(); + String newHash = new String(encodeBase64(bytes)); + return newHash; + } + + private String getItemName(RecordMetadata record) { + return record.getVersionPath(record.getLatestVersion()); + } + + private String getItemName(RecordMetadata record, Long version) { + return record.getVersionPath(version); + } + + @Override + public void delete(RecordMetadata record) { + String itemName = getItemName(record).replace(":", "-"); + try { + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete record.", + e.getMessage()); + } + } + + @Override + public void deleteVersion(RecordMetadata record, Long version) { + String itemName = getItemName(record, version).replace(":", "-"); + try { + if (!record.hasVersion()) { + LOGGER.warn(String.format("Record %s does not have versions available", record.getId())); + } + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to delete version.", + e.getMessage()); + } + } + + @Override + public boolean hasAccess(RecordMetadata... records) { + for (RecordMetadata record : records) { + if (!record.getStatus().equals(RecordState.active)) { + return false; + } + } + return true; + } + + @Override + public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) { + String itemName = getItemName(record, version).replace(":", "-"); + String msg = String + .format("Record with id '%s' does not exist, version: %s", record.getId(), version); + InputStream stream; + try { + stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(itemName) + .build()); + if (stream == null) { + LOGGER.warn(msg); + } else { + return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", + e.getMessage()); + } + throw new AppException(HttpStatus.SC_NOT_FOUND, "Record not found", msg); + } + + @Override + public Map<String, String> read(Map<String, String> objects) { + // key -> record id + // value -> record version path + Map<String, String> map = new HashMap<>(); + for (Map.Entry<String, String> record : objects.entrySet()) { + String[] tokens = record.getValue().split("/"); + String key = tokens[tokens.length - 2]; + try { + InputStream stream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(minIoConfigProperties.getMinIoBucketRecordName()) + .object(record.getValue().replace(":", "-")) + .build()); + if (stream != null) { + String result = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + map.put(key, result); + } + } catch (Exception e) { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Failed to get object.", + e.getMessage()); + } + } + return map; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java new file mode 100644 index 0000000000000000000000000000000000000000..590ab40cbfb784e5c1ec47d941de0551c218620c --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/app/StorageReferenceApplication.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.app; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; + +@SpringBootApplication(exclude = {MongoAutoConfiguration.class}) +@ComponentScan({"org.opengroup.osdu"}) +public class StorageReferenceApplication { + + public static void main(String[] args) { + + SpringApplication.run(StorageReferenceApplication.class, args); + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java new file mode 100644 index 0000000000000000000000000000000000000000..60eaab2f3a2b6233435aae05249f083145761f0e --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/GroupCache.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.model.entitlements.Groups; +import org.opengroup.osdu.storage.provider.reference.config.RedisConfigProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class GroupCache extends RedisCache<String, Groups> { + + @Autowired + public GroupCache(RedisConfigProperties redisConfigProperties) { + super(redisConfigProperties.getGcpRedisHost(), + Integer.parseInt(redisConfigProperties.getGcpRedisPort()), + Integer.parseInt(redisConfigProperties.getGcpRedisExpTime()), String.class, Groups.class); + } +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java new file mode 100644 index 0000000000000000000000000000000000000000..0388780bce1cbeedea7f0e2d79f52f39c78dc9f9 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/LegalTagCache.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.cache.MultiTenantCache; +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component("LegalTagCache") +public class LegalTagCache implements ICache<String, String> { + + @Autowired + private TenantInfo tenant; + + private final MultiTenantCache<String> caches; + + public LegalTagCache() { + this.caches = new MultiTenantCache<>( + new VmCache(60 * 60, 1000)); + } + + @Override + public void put(String key, String val) { + this.partitionCache().put(key, val); + } + + @Override + public String get(String key) { + return this.partitionCache().get(key); + } + + @Override + public void delete(String key) { + this.partitionCache().delete(key); + } + + @Override + public void clearAll() { + this.partitionCache().clearAll(); + } + + private ICache<String, String> partitionCache() { + return this.caches.get(String.format("%s:legalTag", this.tenant)); + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java new file mode 100644 index 0000000000000000000000000000000000000000..e95694fdc2b53323b26a7574d68b933251c0fcc4 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/cache/SchemaCache.java @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.cache; + +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.springframework.stereotype.Component; + +@Component +public class SchemaCache extends VmCache<String, Schema> { + + public SchemaCache() { + super(5 * 60, 1000); + } +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..d5000b7759c19480dc41ed8cc398edfe93ce4dd4 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MinIoConfigProperties.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MinIoConfigProperties { + + private String minIoSignedUrlExpirationDays; + private String minIoEndpointUrl; + private String minIoAccessKey; + private String minIoSecretKey; + private String minIoRegion; + private String minIoPrefix; + private String minIoBucketRecordName; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..5438065ea5c815897332bf30743d8b1e1a6dbeb6 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/MongoDBConfigProperties.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class MongoDBConfigProperties { + + private String mongoDbUrl; + private String mongoDbUser; + private String mongoDbPassword; + private String mongoDbName; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..f72efd6bc85c193710c07662a50a0a2a2ddf2734 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RabbitMqConfigProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RabbitMqConfigProperties { + + private String mbRabbitMqUri; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..76455e4902d160fa80a5d3541e58b75238cc509a --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/config/RedisConfigProperties.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties +@Getter +@Setter +public class RedisConfigProperties { + + private String gcpRedisPort; + private String gcpRedisHost; + private String gcpRedisExpTime; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..0e54767847aed28531ba3ef72e663e5239ded10f --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/RabbitMQFactoryImpl.java @@ -0,0 +1,88 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.di; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; +import javax.annotation.PostConstruct; +import org.opengroup.osdu.storage.provider.reference.config.RabbitMqConfigProperties; +import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Component +public class RabbitMQFactoryImpl implements IMessageFactory { + + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class); + private final RabbitMqConfigProperties rabbitMqConfigProperties; + + private Channel channel; + + @Autowired + public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) { + this.rabbitMqConfigProperties = rabbitMqConfigProperties; + } + + @PostConstruct + private void init() { + ConnectionFactory factory = new ConnectionFactory(); + try { + String mbRabbitMqUri = rabbitMqConfigProperties.getMbRabbitMqUri(); + LOG.debug(String.format("RabbitMQ Uri = %s", mbRabbitMqUri)); + factory.setUri(mbRabbitMqUri); + factory.setAutomaticRecoveryEnabled(true); + Connection conn = factory.newConnection(); + this.channel = conn.createChannel(); + LOG.debug("RabbitMQ Channel was created."); + for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) { + channel.queueDeclare(queue, true, false, false, null); + LOG.debug(String.format("Queue [ %s ] was declared.", queue)); + } + } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) { + LOG.error(e.getMessage(), e); + } + } + + @Override + public void sendMessage(String msg) { + sendMessage(DEFAULT_QUEUE_NAME, msg); + } + + @Override + public void sendMessage(String queueName, String msg) { + String queueNameWithPrefix = queueName; + try { + channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes()); + LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix)); + } catch (IOException e) { + LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix)); + LOG.error(e.getMessage(), e); + } + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..4307d1ca97e10c4dbc6a431d6a04efb7bf36c4d3 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/di/TenantFactoryImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.di; + +import static java.util.Objects.isNull; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.opengroup.osdu.core.common.cache.ICache; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TenantFactoryImpl implements ITenantFactory { + + private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class); + public static final String TENANT_INFO = "tenantinfo"; + public static final String MAIN_DATABASE = "main"; + + @Autowired + private MongoDdmsClient mongoClient; + + private Map<String, TenantInfo> tenants; + + public boolean exists(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.containsKey(tenantName); + } + + public TenantInfo getTenantInfo(String tenantName) { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.get(tenantName); + } + + public Collection<TenantInfo> listTenantInfo() { + if (this.tenants == null) { + initTenants(); + } + return this.tenants.values(); + } + + public <V> ICache<String, V> createCache(String tenantName, String host, int port, + int expireTimeSeconds, Class<V> classOfV) { + return null; + } + + public void flushCache() { + } + + private void initTenants() { + this.tenants = new HashMap<>(); + MongoCollection<Document> mongoCollection = mongoClient + .getMongoCollection(MAIN_DATABASE, TENANT_INFO); + FindIterable<Document> results = mongoCollection.find(); + if (isNull(results) || isNull(results.first())) { + LOG.error(String.format("Collection \'%s\' is empty.", results)); + } + for (Document document : results) { + TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class); + ObjectId id = (ObjectId) document.get("_id"); + tenantInfo.setId((long) id.getCounter()); + tenantInfo.setCrmAccountIds((ArrayList<String>) document.get("crmAccountID")); + this.tenants.put(tenantInfo.getName(), tenantInfo); + } + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..c4db1f8185a7251dcb79ed045f5e02ed75832831 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/factory/CloudObjectStorageFactory.java @@ -0,0 +1,61 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.factory; + +import io.minio.MinioClient; +import javax.annotation.PostConstruct; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component +@Lazy +public class CloudObjectStorageFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(CloudObjectStorageFactory.class); + private final MinIoConfigProperties minIoConfigProperties; + + private MinioClient minioClient; + + @Autowired + public CloudObjectStorageFactory(MinIoConfigProperties minIoConfigProperties) { + this.minIoConfigProperties = minIoConfigProperties; + } + + @PostConstruct + public void init() { + minioClient = MinioClient.builder() + .endpoint(minIoConfigProperties.getMinIoEndpointUrl()) + .credentials(minIoConfigProperties.getMinIoAccessKey(), + minIoConfigProperties.getMinIoSecretKey()) + .region(minIoConfigProperties.getMinIoRegion()) + .build(); + LOGGER.info("Minio client initialized"); + } + + public MinioClient getClient() { + return this.minioClient; + } + + public void setMinioClient(MinioClient minioClient) { + this.minioClient = minioClient; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..fe78a970cc470d01059a93b6552304040e9f4261 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/jobs/LegalComplianceChangeServiceImpl.java @@ -0,0 +1,137 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.jobs; + +import static java.util.Collections.singletonList; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.OperationType; +import org.opengroup.osdu.core.common.model.legal.LegalCompliance; +import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceChangeInfo; +import org.opengroup.osdu.core.common.model.legal.jobs.ILegalComplianceChangeService; +import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChanged; +import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection; +import org.opengroup.osdu.core.common.model.storage.PubSubInfo; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.opengroup.osdu.storage.logging.StorageAuditLogger; +import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.reference.cache.LegalTagCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class LegalComplianceChangeServiceImpl implements ILegalComplianceChangeService { + + private static final Logger LOG = LoggerFactory.getLogger(LegalComplianceChangeServiceImpl.class); + + @Autowired + private IRecordsMetadataRepository recordsRepo; + + @Autowired + private IMessageBus pubSubClient; + + @Autowired + private StorageAuditLogger auditLogger; + + @Autowired + private LegalTagCache legalTagCache; + + @Override + public Map<String, LegalCompliance> updateComplianceOnRecords( + LegalTagChangedCollection legalTagsChanged, + DpsHeaders headers) { + Map<String, LegalCompliance> output = new HashMap<>(); + + for (LegalTagChanged lt : legalTagsChanged.getStatusChangedTags()) { + + ComplianceChangeInfo complianceChangeInfo = this.getComplianceChangeInfo(lt); + if (complianceChangeInfo == null) { + continue; + } + + String cursor = null; + do { + //TODO replace with the new method queryByLegal + AbstractMap.SimpleEntry<String, List<RecordMetadata>> results = this.recordsRepo + .queryByLegalTagName(lt.getChangedTagName(), 500, cursor); + cursor = results.getKey(); + + if (results.getValue() != null && !results.getValue().isEmpty()) { + List<RecordMetadata> recordsMetadata = results.getValue(); + PubSubInfo[] pubSubInfos = this + .updateComplianceStatus(complianceChangeInfo, recordsMetadata, output); + this.recordsRepo.createOrUpdate(recordsMetadata); + StringBuilder recordsId = new StringBuilder(); + for (RecordMetadata recordMetadata : recordsMetadata) { + recordsId.append(", ").append(recordMetadata.getId()); + } + this.auditLogger.updateRecordsComplianceStateSuccess( + singletonList("[" + recordsId.substring(2) + "]")); + + this.pubSubClient.publishMessage(headers, pubSubInfos); + } + } while (cursor != null); + } + + return output; + } + + private PubSubInfo[] updateComplianceStatus(ComplianceChangeInfo complianceChangeInfo, + List<RecordMetadata> recordMetadata, Map<String, LegalCompliance> output) { + + PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()]; + + int i = 0; + for (RecordMetadata rm : recordMetadata) { + rm.getLegal().setStatus(complianceChangeInfo.getNewState()); + rm.setStatus(complianceChangeInfo.getNewRecordState()); + pubsubInfo[i] = new PubSubInfo(rm.getId(), rm.getKind(), + complianceChangeInfo.getPubSubEvent()); + output.put(rm.getId(), complianceChangeInfo.getNewState()); + i++; + } + + return pubsubInfo; + } + + private ComplianceChangeInfo getComplianceChangeInfo(LegalTagChanged lt) { + ComplianceChangeInfo output = null; + + if (lt.getChangedTagStatus().equalsIgnoreCase("compliant")) { + output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.create, + RecordState.active); + } else if (lt.getChangedTagStatus().equalsIgnoreCase("incompliant")) { + this.legalTagCache.delete(lt.getChangedTagName()); + output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, + RecordState.deleted); + } else { + LOG.warn(String.format("Unknown LegalTag compliance status received %s %s", + lt.getChangedTagStatus(), lt.getChangedTagName())); + } + + return output; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..4db9ec80f07571af4aa18dcd769fad766d02a786 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/messagebus/IMessageFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.messagebus; + +public interface IMessageFactory { + + String DEFAULT_QUEUE_NAME = "records"; + String LEGAL_QUEUE_NAME = "legal"; + String INDEXER_QUEUE_NAME = "indexer"; + + void sendMessage(String msg); + + void sendMessage(String queueName, String msg); +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java new file mode 100644 index 0000000000000000000000000000000000000000..0038bb9ec440174b92b94aececc64887cd4f7a86 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/RecordMetadataDocument.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.ArrayList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.opengroup.osdu.core.common.model.entitlements.Acl; +import org.opengroup.osdu.core.common.model.legal.Legal; +import org.opengroup.osdu.core.common.model.storage.RecordAncestry; +import org.opengroup.osdu.core.common.model.storage.RecordState; +import org.springframework.data.mongodb.core.mapping.Document; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Document(collection = "StorageRecord") +public class RecordMetadataDocument { + + private String id; + private String kind; + private Acl acl; + private Legal legal; + private RecordAncestry ancestry; + private List<String> gcsVersionPaths = new ArrayList(); + private RecordState status; + private String user; + private Long createTime; + private String modifyUser; + private Long modifyTime; +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java new file mode 100644 index 0000000000000000000000000000000000000000..65bfe11a28ee60b0e994b44acdc6e5f8cd41aca2 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/SchemaDocument.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.Map; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.opengroup.osdu.core.common.model.storage.SchemaItem; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "StorageSchema") +public class SchemaDocument { + + private String kind; + private String rev; + private String user; + private SchemaItem[] schema; + private Map<String, Object> extension; + + + public SchemaDocument(Schema schema, String user) { + this.setKind(schema.getKind()); + this.setExtension(schema.getExt()); + this.setSchema(schema.getSchema()); + this.setUser(user); + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + public Map<String, Object> getExtension() { + return extension; + } + + public void setExtension(Map<String, Object> extension) { + this.extension = extension; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public SchemaItem[] getSchema() { + return schema; + } + + public void setSchema(SchemaItem[] schemaItems) { + this.schema = schemaItems; + } + +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java new file mode 100644 index 0000000000000000000000000000000000000000..ae2ea414f69d13b533b80af786fb2e054ded5fea --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/model/TenantInfoDocument.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.model; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.mongodb.core.mapping.Document; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Document(collection = "TenantInfo") +public class TenantInfoDocument { + + private String id; + private List<String> groups; +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java new file mode 100644 index 0000000000000000000000000000000000000000..accf7016d2bbf026367ad6b318c6dd730bee38de --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/persistence/MongoDdmsClient.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.persistence; + +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.opengroup.osdu.storage.provider.reference.util.MongoClientHandler; +import org.springframework.stereotype.Component; + +@Component +public class MongoDdmsClient { + + private MongoClientHandler mongoClientHandler; + + public MongoDdmsClient(MongoClientHandler mongoClientHandler) { + this.mongoClientHandler = mongoClientHandler; + } + + public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) { + return mongoClientHandler.getMongoClient().getDatabase(dbName) + .getCollection(collectionName); + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..8c165109bd67bebbc3ebe1d5c9f643938f6e7633 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/QueryRepositoryImpl.java @@ -0,0 +1,95 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.RECORD_STORAGE; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_STORAGE; + +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; +import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class QueryRepositoryImpl implements IQueryRepository { + private MongoDdmsClient mongoDdmsClient; + + @Autowired + public QueryRepositoryImpl(MongoDdmsClient mongoDdmsClient) { + this.mongoDdmsClient = mongoDdmsClient; + } + + @Override + public DatastoreQueryResult getAllKinds(Integer limit, String cursor) { + int numRecords = PAGE_SIZE; + if (limit != null) { + numRecords = limit > 0 ? limit : PAGE_SIZE; + } + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + FindIterable<Document> results = mongoCollection.find() + .limit(numRecords); + List<String> kinds = new ArrayList<>(); + for (Document document : results) { + kinds.add(document.get("kind").toString()); + } + return new DatastoreQueryResult(cursor, kinds); + } + + @Override + public DatastoreQueryResult getAllRecordIdsFromKind( + String kind, Integer limit, String cursor) { + boolean paginated = false; + + int numRecords = PAGE_SIZE; + if (Objects.nonNull(limit)) { + numRecords = limit > 0 ? limit : PAGE_SIZE; + paginated = true; + } + + if (cursor != null && !cursor.isEmpty()) { + paginated = true; + } + + DatastoreQueryResult dqr = new DatastoreQueryResult(); + List<String> ids = new ArrayList(); + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, RECORD_STORAGE); + FindIterable<Document> results = mongoCollection.find(eq("kind", kind)) + .limit(numRecords); + for (Document document : results) { + ids.add(document.get("id").toString()); + } + dqr.setResults(ids); + dqr.setCursor(cursor); + return dqr; + } + +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..b69f712c2b11ca753264b5987ef01ac8e470afe6 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/RecordsMetadataRepositoryImpl.java @@ -0,0 +1,174 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.util.JSON.serialize; +import static org.opengroup.osdu.storage.provider.reference.repository.SchemaRepositoryImpl.SCHEMA_DATABASE; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.UpdateOptions; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.legal.LegalCompliance; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository; +import org.opengroup.osdu.storage.provider.reference.model.RecordMetadataDocument; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository<String> { + + public static final String STORAGE_RECORD = "StorageRecord"; + private final MongoDdmsClient mongoDdmsClient; + + @Autowired + public RecordsMetadataRepositoryImpl(MongoDdmsClient mongoDdmsClient) { + this.mongoDdmsClient = mongoDdmsClient; + } + + @Override + public List<RecordMetadata> createOrUpdate(List<RecordMetadata> recordsMetadata) { + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + if (Objects.nonNull(recordsMetadata)) { + for (RecordMetadata recordMetadata : recordsMetadata) { + RecordMetadataDocument recordMetadataDocument = convertToRecordMetadataDocument( + recordMetadata); + mongoCollection.replaceOne(eq("id", recordMetadataDocument.getId()), + Document.parse(new Gson().toJson(recordMetadataDocument)), + (new UpdateOptions()).upsert(true)); + } + } + return recordsMetadata; + } + + @Override + public void delete(String id) { + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + mongoCollection.deleteOne(eq("id", id)); + } + + @Override + public RecordMetadata get(String id) { + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + Document doc = mongoCollection.find(eq("id", id)).first(); + if (Objects.isNull(doc)) { + return null; + } + RecordMetadataDocument recordMetadataDocument = new Gson() + .fromJson(serialize(doc), RecordMetadataDocument.class); + return convertToRecordMetadata(recordMetadataDocument); + } + + @Override + public Map<String, RecordMetadata> get(List<String> ids) { + Map<String, RecordMetadata> output = new HashMap<>(); + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + for (String id : ids) { + Document document = mongoCollection.find(eq("id", id)).first(); + RecordMetadataDocument recordMetadataDocument = null; + if (Objects.nonNull(document)) { + recordMetadataDocument = new Gson() + .fromJson(serialize(document), RecordMetadataDocument.class); + } + RecordMetadata rmd = convertToRecordMetadata(recordMetadataDocument); + if (Objects.isNull(rmd)) { + continue; + } + output.put(id, rmd); + } + return output; + } + + @Override + public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName( + String legalTagName, int limit, String cursor) { + MongoCollection<Document> mongoCollection = mongoDdmsClient + .getMongoCollection(SCHEMA_DATABASE, STORAGE_RECORD); + List<RecordMetadata> outputRecords = new ArrayList<>(); + FindIterable<Document> results = mongoCollection.find().skip(limit * (limit - 1)).limit(limit); + for (Document document : results) { + RecordMetadataDocument recordMetadataDocument = new Gson() + .fromJson(serialize(document), RecordMetadataDocument.class); + if (Objects.nonNull(recordMetadataDocument)) { + if (recordMetadataDocument.getLegal().getLegaltags().contains(legalTagName)) { + RecordMetadata recordMetadata = convertToRecordMetadata(recordMetadataDocument); + outputRecords.add(recordMetadata); + } + } + } + return new AbstractMap.SimpleEntry<>(cursor, outputRecords); + } + + @Override + public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegal(String legalTagName, + LegalCompliance status, int limit) { + return null; + } + + private RecordMetadataDocument convertToRecordMetadataDocument(RecordMetadata recordMetadata) { + RecordMetadataDocument recordMetadataDocument = new RecordMetadataDocument(); + recordMetadataDocument.setId(recordMetadata.getId()); + recordMetadataDocument.setAcl(recordMetadata.getAcl()); + recordMetadataDocument.setAncestry(recordMetadata.getAncestry()); + recordMetadataDocument.setCreateTime(recordMetadata.getCreateTime()); + recordMetadataDocument.setModifyTime(recordMetadata.getModifyTime()); + recordMetadataDocument.setGcsVersionPaths(recordMetadata.getGcsVersionPaths()); + recordMetadataDocument.setKind(recordMetadata.getKind()); + recordMetadataDocument.setLegal(recordMetadata.getLegal()); + recordMetadataDocument.setModifyUser(recordMetadata.getModifyUser()); + recordMetadataDocument.setStatus(recordMetadata.getStatus()); + recordMetadataDocument.setUser(recordMetadata.getUser()); + + return recordMetadataDocument; + } + + private RecordMetadata convertToRecordMetadata(RecordMetadataDocument recordMetadataDocument) { + if (Objects.isNull(recordMetadataDocument)) { + return null; + } + RecordMetadata recordMetadata = new RecordMetadata(); + recordMetadata.setId(recordMetadataDocument.getId()); + recordMetadata.setAcl(recordMetadataDocument.getAcl()); + recordMetadata.setAncestry(recordMetadataDocument.getAncestry()); + recordMetadata.setCreateTime(recordMetadataDocument.getCreateTime()); + recordMetadata.setModifyTime(recordMetadataDocument.getModifyTime()); + recordMetadata.setGcsVersionPaths(recordMetadataDocument.getGcsVersionPaths()); + recordMetadata.setKind(recordMetadataDocument.getKind()); + recordMetadata.setLegal(recordMetadataDocument.getLegal()); + recordMetadata.setModifyUser(recordMetadataDocument.getModifyUser()); + recordMetadata.setStatus(recordMetadataDocument.getStatus()); + recordMetadata.setUser(recordMetadataDocument.getUser()); + + return recordMetadata; + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..c8ae6fd728a9209dccd29816a59b023ac57fb5f3 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/repository/SchemaRepositoryImpl.java @@ -0,0 +1,94 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.repository; + +import static com.mongodb.client.model.Filters.eq; + +import com.google.gson.Gson; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import java.util.Objects; +import org.apache.http.HttpStatus; +import org.bson.Document; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.Schema; +import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository; +import org.opengroup.osdu.storage.provider.reference.model.SchemaDocument; +import org.opengroup.osdu.storage.provider.reference.persistence.MongoDdmsClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class SchemaRepositoryImpl implements ISchemaRepository { + + private static final Logger logger = LoggerFactory.getLogger(SchemaRepositoryImpl.class); + public static final String SCHEMA_STORAGE = "SchemaStorage"; + public static final String RECORD_STORAGE = "StorageRecord"; + public static final String SCHEMA_DATABASE = "schema"; + + private MongoDdmsClient mongoClient; + + @Autowired + public SchemaRepositoryImpl(MongoDdmsClient mongoClient) { + this.mongoClient = mongoClient; + } + + @Override + public void add(Schema schema, String user) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + String kind = schema.getKind(); + FindIterable<Document> results = collection.find(eq("kind", kind)); + if (Objects.nonNull(results) && Objects.nonNull(results.first())) { + throw new IllegalArgumentException("Schema " + kind + " already exist. Can't create again."); + } + SchemaDocument schemaDocument = new SchemaDocument(schema, user); + collection.insertOne(Document.parse(new Gson().toJson(schemaDocument))); + } + + @Override + public Schema get(String kind) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + Document record = (Document) collection.find(eq("kind", kind)).first(); + if (Objects.isNull(record)) { + throw new AppException( + HttpStatus.SC_NOT_FOUND, "Not found", + String.format("Schema with id %s does not exist.", kind)); + } + SchemaDocument schemaDocument = new Gson().fromJson(record.toJson(), SchemaDocument.class); + return convertToSchemaEntity(schemaDocument); + } + + @Override + public void delete(String kind) { + MongoCollection collection = this.mongoClient + .getMongoCollection(SCHEMA_DATABASE, SCHEMA_STORAGE); + collection.deleteOne(eq("kind", kind)); + } + + private Schema convertToSchemaEntity(SchemaDocument schemaDocument) { + Schema schema = new Schema(); + schema.setKind(schemaDocument.getKind()); + schema.setExt(schemaDocument.getExtension()); + schema.setSchema(schemaDocument.getSchema()); + return schema; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..ff2a17f6e2375df559bba57ad58dd244c3deacc3 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/BasicAuthSecurityConfig.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.security; + +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity http) throws Exception { + http + .httpBasic().disable() + .csrf().disable(); + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java new file mode 100644 index 0000000000000000000000000000000000000000..76eba1e23d0db213a706e9baa887adca49ee2bfd --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/security/WhoamiController.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.security; + +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +public class WhoamiController { + + @RequestMapping(value = {"/", "/whoami"}) + @ResponseBody + public String whoami() { + final Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + String userName = auth.getName(); + String roles = String.valueOf(auth.getAuthorities()); + String details = String.valueOf(auth.getPrincipal()); + + return "user: " + userName + "<BR>" + + "roles: " + roles + "<BR>" + + "details: " + details; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..7d073db5bba12d42db19b9da8badfc37560898d5 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/BatchServiceReferenceImpl.java @@ -0,0 +1,58 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.service; + +import static java.util.Collections.singletonList; + +import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult; +import org.opengroup.osdu.storage.logging.StorageAuditLogger; +import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository; +import org.opengroup.osdu.storage.service.BatchServiceImpl; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class BatchServiceReferenceImpl extends BatchServiceImpl { + + private final StorageAuditLogger auditLogger; + private final IQueryRepository queryRepository; + + @Autowired + public BatchServiceReferenceImpl(StorageAuditLogger auditLogger, + IQueryRepository queryRepository) { + this.auditLogger = auditLogger; + this.queryRepository = queryRepository; + } + + @Override + public DatastoreQueryResult getAllKinds(String cursor, Integer limit) { + DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor); + this.auditLogger.readAllKindsSuccess(result.getResults()); + return result; + } + + @Override + public DatastoreQueryResult getAllRecords(String cursor, String kind, Integer limit) { + DatastoreQueryResult result = this.queryRepository.getAllRecordIdsFromKind(kind, limit, cursor); + if (!result.getResults().isEmpty()) { + this.auditLogger.readAllRecordsOfGivenKindSuccess(singletonList(kind)); + } + return result; + } + +} \ No newline at end of file diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..409e01348f59e93161f76fb7d6a816fc23cd8418 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/service/MessageBusImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.service; + +import com.google.gson.Gson; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.storage.PubSubInfo; +import org.opengroup.osdu.storage.provider.interfaces.IMessageBus; +import org.opengroup.osdu.storage.provider.reference.messagebus.IMessageFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MessageBusImpl implements IMessageBus { + + private final IMessageFactory messageQueue; + + @Autowired + public MessageBusImpl(IMessageFactory messageQueue) { + this.messageQueue = messageQueue; + } + + public void publishMessage(DpsHeaders headers, PubSubInfo... messages) { + final int BATCH_SIZE = 50; + Map<String, String> message = new HashMap<>(); + Gson gson = new Gson(); + + for (int i = 0; i < messages.length; i += BATCH_SIZE) { + PubSubInfo[] batch = Arrays + .copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE)); + + String json = gson.toJson(batch); + message.put("data", json); + message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + message.put(DpsHeaders.AUTHORIZATION, headers.getAuthorization()); + messageQueue.sendMessage(gson.toJson(message)); + } + } +} + diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..7bbb0a9a9f7f139bf18d9fc197de909b76838842 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/MongoClientHandler.java @@ -0,0 +1,80 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.util; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; +import org.opengroup.osdu.storage.provider.reference.config.MongoDBConfigProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MongoClientHandler { + + private static final Logger LOG = LoggerFactory.getLogger(MongoClientHandler.class); + private static final String MONGO_PREFIX = "mongodb://"; + private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000"; + + private com.mongodb.client.MongoClient mongoClient = null; + private MongoDBConfigProperties mongoDBConfigProperties; + + private MongoClient getOrInitMongoClient() throws RuntimeException { + if (mongoClient != null) { + return mongoClient; + } + + final String connectionString = String.format("%s%s:%s@%s/?%s", + MONGO_PREFIX, + mongoDBConfigProperties.getMongoDbUser(), + mongoDBConfigProperties.getMongoDbPassword(), + mongoDBConfigProperties.getMongoDbUrl(), + MONGO_OPTIONS); + ConnectionString connString = new ConnectionString(connectionString); + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connString) + .retryWrites(true) + .build(); + try { + mongoClient = MongoClients.create(settings); + } catch (Exception ex) { + LOG.error("Error connecting MongoDB", ex); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB", + ex.getMessage(), ex); + } + return mongoClient; + } + + public MongoClient getMongoClient() { + if (mongoClient == null) { + getOrInitMongoClient(); + } + return mongoClient; + } + + @Autowired + public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) { + this.mongoDBConfigProperties = mongoDBConfigProperties; + } +} diff --git a/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..07adc6bcb0e8386b158f99de4e2198a274a98086 --- /dev/null +++ b/provider/storage-reference/src/main/java/org/opengroup/osdu/storage/provider/reference/util/ServiceAccountJwtClientImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.util; + +import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient { + + @Override + public String getIdToken(String tenantName) { + return "dont-have-one"; + } +} diff --git a/provider/storage-reference/src/main/resources/application.properties b/provider/storage-reference/src/main/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..1e7c8dc1b65a3415ded022c8189109b1272ed606 --- /dev/null +++ b/provider/storage-reference/src/main/resources/application.properties @@ -0,0 +1,27 @@ +LOG_PREFIX=storage +server.servlet.contextPath=/api/storage/v2/ +logging.level.org.springframework.web=DEBUG +server.port=8080 +JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M +JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 + +AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 +LEGALTAG_API=https://os-legal-ibm/api/legal/v1 + +mongo-db-url=localhost:27017 +mongo-db-user= +mongo-db-password= + +#amqp://guest:guest@127.0.0.1:5672/%2F by default +mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672/%2F + +minio-endpoint-url=http://127.0.0.1:9000 +minio-access-key= +minio-secret-key= +minio-region= +minio-prefix=local-dev +minio-bucket-record-name=record-bucket + +gcp-redis-host=localhost +gcp-redis-port=6379 +gcp-redis-exp-time=10 \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f4fb1e2f387e6e4c6431e0fde9e2c186286fa271 --- /dev/null +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/CloudStorageImplTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.minio.MinioClient; +import java.util.HashMap; +import java.util.Map; +import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.storage.Record; +import org.opengroup.osdu.core.common.model.storage.RecordData; +import org.opengroup.osdu.core.common.model.storage.RecordMetadata; +import org.opengroup.osdu.core.common.model.storage.RecordProcessing; +import org.opengroup.osdu.storage.provider.reference.config.MinIoConfigProperties; +import org.opengroup.osdu.storage.provider.reference.factory.CloudObjectStorageFactory; + +@RunWith(MockitoJUnitRunner.class) +public class CloudStorageImplTest extends TestCase { + + private RecordProcessing[] recordProcessingArray = new RecordProcessing[1]; + private RecordProcessing recordProcessing; + private RecordMetadata recordMetadata; + private String bucketName = "osdu-sample-osdu-file"; + + @Mock + private CloudObjectStorageFactory factory; + + @Mock + private MinioClient minioClient; + + @Mock + private MinIoConfigProperties minIoConfigProperties; + + @InjectMocks + private CloudStorageImpl cloudStorage; + + @Before + public void setup() { + recordMetadata = new RecordMetadata(); + recordMetadata.setKind("test-record-id"); + recordMetadata.setId("test-record-id"); + recordMetadata.addGcsPath(1); + recordMetadata.addGcsPath(2); + + recordProcessing = new RecordProcessing(); + recordProcessing.setRecordMetadata(recordMetadata); + + Record record = new Record(); + record.setId("test-record-id"); + Map<String, Object> data = new HashMap<>(); + data.put("test-data", new Object()); + record.setData(data); + + RecordData recordData = new RecordData(record); + recordProcessing.setRecordData(recordData); + recordProcessingArray[0] = recordProcessing; + } + + @Test + public void write_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + cloudStorage.write(recordProcessingArray); + + verify(minioClient, times(1)).putObject(any()); + } + + @Test + public void delete_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + cloudStorage.delete(recordMetadata); + + verify(minioClient, times(1)).removeObject(any()); + } + + @Test + public void deleteVersion_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + cloudStorage.deleteVersion(recordMetadata, 1L); + + verify(minioClient, times(1)).removeObject(any()); + } + + @Test(expected = AppException.class) + public void read_with_version_test() throws Exception { + cloudStorage.read(recordMetadata, 1L, false); + + verify(minioClient, times(1)).getObject(any()); + } + + @Test + public void read_test() throws Exception { + when(minIoConfigProperties.getMinIoBucketRecordName()).thenReturn(bucketName); + + Map<String, String> map = new HashMap<>(); + map.put("common:welldb:1", "opendes:ds:mytest1:1.0.0/common:welldb:123456/1603618609515093"); + cloudStorage.read(map); + + verify(minioClient, times(1)).getObject(any()); + } +} \ No newline at end of file diff --git a/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5fd9839cec5ed96ec3d6221192e30e14e882c846 --- /dev/null +++ b/provider/storage-reference/src/test/java/org/opengroup/osdu/storage/provider/reference/context/ProviderSpringContextTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Google LLC + * Copyright 2021 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.storage.provider.reference.context; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@Ignore +@RunWith(SpringRunner.class) +@SpringBootTest +public class ProviderSpringContextTest { + + @Test + public void load_spring_context() { + + } +} diff --git a/provider/storage-reference/src/test/resources/application.properties b/provider/storage-reference/src/test/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..7e2bae18838af6c1a422575c500001f43a800481 --- /dev/null +++ b/provider/storage-reference/src/test/resources/application.properties @@ -0,0 +1,28 @@ +LOG_PREFIX=storage + +server.servlet.contextPath=/api/storage/v2/ +logging.level.org.springframework.web=DEBUG +server.port=8080 +JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M +JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 + +AUTHORIZE_API=https://os-entitlements:8080/api/entitlements/v1 +LEGALTAG_API=https://os-legal-ibm/api/legal/v1 + +mongo.db.url=localhost:27017 +mongo.db.user=admin +mongo.db.password=admin + +#amqp://guest:guest@127.0.0.1:5672/%2F by default +mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672/%2F + +minio.endpoint.url=http://127.0.0.1:9000 +minio.access.key=adminadmin +minio.secret.key=adminadmin +minio.region=admin +minio.prefix=local-dev +minio.bucket.record.name=record-bucket + +gcp.redis.host=localhost +gcp.redis.port=6379 +gcp.redis.exp.time=10 \ No newline at end of file diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e5375a804253ec0668ef2ebdb63d53c96f15deb --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,12 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: storage-reference +build: + artifacts: + - image: us.gcr.io/osdu-anthos-02/os-storage/anthos-storage-reference + context: ./provider/storage-reference +deploy: + kubectl: + manifests: + - ./provider/storage-reference/kubernetes/deployments/deployment-os-storage-service.yml \ No newline at end of file