From 3d9c5d160f89385b682cbd8c14e72eb17275a0bc Mon Sep 17 00:00:00 2001
From: "Aleh Shubko [EPAM]" <aleh_shubko1@epam.com>
Date: Fri, 22 Jul 2022 16:41:05 +0000
Subject: [PATCH] release MongoDB implementation
---
.fossa.yml | 4 -
.gitlab-ci.yml | 11 +
NOTICE | 4 +
pom.xml | 1 -
.../storage-aws-mongodb/build-aws/Dockerfile | 22 --
.../build-aws/build-info.py | 75 -----
.../build-aws/buildspec.yaml | 99 ------
provider/storage-aws-mongodb/lombok.config | 3 -
.../storage-aws-mongodb/maven/settings.xml | 80 -----
provider/storage-aws-mongodb/pom.xml | 170 ----------
.../provider/mongodb/CloudStorageImpl.java | 293 ------------------
.../provider/mongodb/MessageBusImpl.java | 99 ------
.../mongodb/QueryRepositoryImplMongoDB.java | 100 ------
.../RecordsMetadataRepositoryImplMongoDB.java | 160 ----------
.../mongodb/SchemaRepositoryImplMongoDB.java | 93 ------
.../provider/mongodb/cache/GroupCache.java | 40 ---
.../provider/mongodb/cache/LegalTagCache.java | 65 ----
.../provider/mongodb/cache/SchemaCache.java | 30 --
.../mongodb/config/MongoDBConfiguration.java | 32 --
.../LegalComplianceChangeServiceAWSImpl.java | 139 ---------
.../security/BasicAuthSecurityConfig.java | 31 --
.../mongodb/security/UserAccessService.java | 153 ---------
.../mongodb/security/WhoamiController.java | 38 ---
.../mongodb/service/BatchServiceAWSImpl.java | 73 -----
.../provider/mongodb/util/CacheHelper.java | 28 --
.../LegalTagAssociationDocMongoDB.java | 39 ---
.../documents/RecordMetadataDocMongoDB.java | 57 ----
.../mongodb/documents/SchemaMongoDBDoc.java | 34 --
.../util/s3/GetRecordFromVersionTask.java | 57 ----
.../mongodb/util/s3/GetRecordTask.java | 51 ---
.../mongodb/util/s3/RecordProcessor.java | 47 ---
.../provider/mongodb/util/s3/RecordsUtil.java | 125 --------
.../mongodb/util/s3/S3RecordClient.java | 158 ----------
.../src/main/resources/application.properties | 45 ---
.../provider/mongodb/CloudStorageTest.java | 169 ----------
...galComplianceChangeServiceAWSImplTest.java | 174 -----------
.../provider/mongodb/MessageBusImplTest.java | 110 -------
.../provider/mongodb/S3RecordClientTest.java | 152 ---------
.../mongodb/UserAccessServiceTest.java | 128 --------
.../provider/mongodb/util/Generator.java | 89 ------
.../src/test/resources/application.properties | 35 ---
provider/storage-aws/pom.xml | 21 +-
.../provider/aws/QueryRepositoryImpl.java | 3 +
.../aws/RecordsMetadataRepositoryImpl.java | 3 +
.../provider/aws/mongo/IndexUpdater.java | 61 ++++
.../aws/mongo/MongoDbQueryRepository.java | 99 ++++++
.../MongoDbRecordsMetadataRepository.java | 158 ++++++++++
.../mongo/dto/RecordMetadataMongoDBDto.java} | 17 +-
.../aws/mongo/mongodb/EntityFieldPaths.java} | 18 +-
.../MultiClusteredConfigReaderStorage.java | 27 ++
.../src/main/resources/application.properties | 21 ++
.../provider/aws/api/CloudStorageTest.java | 20 +-
...galComplianceChangeServiceAWSImplTest.java | 2 +-
.../provider/aws/api/MessageBusImplTest.java | 7 +-
.../provider/aws/api/QueryRepositoryTest.java | 9 +-
.../api/RecordsMetadataRepositoryTest.java | 15 +-
.../provider/aws/api/S3RecordClientTest.java | 22 +-
.../aws/api/SchemaRepositoryTest.java | 11 +-
.../aws/api/UserAccessServiceTest.java | 18 +-
.../api/mongo/QueryRepositoryMongoTest.java | 91 ++++++
.../RecordsMetadataRepositoryMongoTest.java | 230 ++++++++++++++
.../configuration/StorageTestConfig.java | 91 ++++++
.../provider/aws/api/mongo/util/DbUtil.java} | 13 +-
.../api/mongo/util/MongoTemplateHelper.java | 53 ++++
.../aws/api/mongo/util/ParentUtil.java | 64 ++++
.../mongo/util/RecordMetadataGenerator.java | 56 ++++
.../src/test/resources/application.properties | 7 +
testing/pom.xml | 1 -
.../build-aws/prepare-dist.sh | 48 ---
.../build-aws/run-tests.sh | 64 ----
testing/storage-test-aws-mongodb/pom.xml | 153 ---------
.../PubsubEndpoint/TestPubsubEndpoint.java | 49 ---
...estPopulateLegalInfoFromParentRecords.java | 50 ---
.../osdu/storage/misc/TestStorageCors.java | 35 ---
.../osdu/storage/misc/TestStress.java | 49 ---
.../query/TestGetQueryInfoIntegration.java | 34 --
.../query/TestGetQueryKindsIntegration.java | 58 ----
.../query/TestGetQueryRecordsIntegration.java | 49 ---
.../TestPostFetchRecordsIntegration.java | 49 ---
.../TestPostQueryRecordsIntegration.java | 49 ---
.../query/TestStorageQuerySuccessful.java | 72 -----
...stDeleteRecordLogicallyAndItsVersions.java | 37 ---
.../records/TestGetRecordsIntegration.java | 50 ---
.../records/TestIngestRecordNotFound.java | 70 -----
.../records/TestLogicalRecordDelete.java | 49 ---
.../records/TestPurgeRecordsIntegration.java | 49 ---
.../TestRecordAccessAuthorization.java | 50 ---
.../records/TestRecordWithNullField.java | 38 ---
.../records/TestRecordsApiAcceptance.java | 50 ---
.../records/TestUpdateRecordsMetadata.java | 37 ---
.../schema/TestCreateSchemaIntegration.java | 36 ---
.../schema/TestStorageSchemaNegative.java | 36 ---
.../osdu/storage/util/AWSCognitoBuilder.java | 26 --
.../osdu/storage/util/AWSCognitoClient.java | 57 ----
.../osdu/storage/util/AWSTestUtils.java | 47 ---
.../osdu/storage/util/SchemaUtil.java | 63 ----
.../src/test/resources/logback-test.xml | 6 -
97 files changed, 1050 insertions(+), 4861 deletions(-)
delete mode 100644 provider/storage-aws-mongodb/build-aws/Dockerfile
delete mode 100644 provider/storage-aws-mongodb/build-aws/build-info.py
delete mode 100644 provider/storage-aws-mongodb/build-aws/buildspec.yaml
delete mode 100644 provider/storage-aws-mongodb/lombok.config
delete mode 100644 provider/storage-aws-mongodb/maven/settings.xml
delete mode 100644 provider/storage-aws-mongodb/pom.xml
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageImpl.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImpl.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/QueryRepositoryImplMongoDB.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/RecordsMetadataRepositoryImplMongoDB.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SchemaRepositoryImplMongoDB.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/GroupCache.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/SchemaCache.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/config/MongoDBConfiguration.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/BasicAuthSecurityConfig.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/UserAccessService.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/WhoamiController.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/service/BatchServiceAWSImpl.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/CacheHelper.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/LegalTagAssociationDocMongoDB.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/RecordMetadataDocMongoDB.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/SchemaMongoDBDoc.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordFromVersionTask.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordTask.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordProcessor.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordsUtil.java
delete mode 100644 provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/S3RecordClient.java
delete mode 100644 provider/storage-aws-mongodb/src/main/resources/application.properties
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageTest.java
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImplTest.java
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/S3RecordClientTest.java
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/UserAccessServiceTest.java
delete mode 100644 provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/util/Generator.java
delete mode 100644 provider/storage-aws-mongodb/src/test/resources/application.properties
create mode 100644 provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/IndexUpdater.java
create mode 100644 provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbQueryRepository.java
create mode 100644 provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbRecordsMetadataRepository.java
rename provider/{storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/ServiceAccountJwtClientImpl.java => storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/dto/RecordMetadataMongoDBDto.java} (53%)
rename provider/{storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SomeBasicInterfaceImpl.java => storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/EntityFieldPaths.java} (57%)
create mode 100644 provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/QueryRepositoryMongoTest.java
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/RecordsMetadataRepositoryMongoTest.java
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/configuration/StorageTestConfig.java
rename provider/{storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/CallableResult.java => storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/DbUtil.java} (60%)
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/MongoTemplateHelper.java
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/ParentUtil.java
create mode 100644 provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/RecordMetadataGenerator.java
create mode 100644 provider/storage-aws/src/test/resources/application.properties
delete mode 100644 testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh
delete mode 100644 testing/storage-test-aws-mongodb/build-aws/run-tests.sh
delete mode 100644 testing/storage-test-aws-mongodb/pom.xml
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/PubsubEndpoint/TestPubsubEndpoint.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/legal/TestPopulateLegalInfoFromParentRecords.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStorageCors.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStress.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryInfoIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryKindsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryRecordsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostFetchRecordsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostQueryRecordsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestStorageQuerySuccessful.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestDeleteRecordLogicallyAndItsVersions.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestGetRecordsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestIngestRecordNotFound.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestLogicalRecordDelete.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestPurgeRecordsIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordWithNullField.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestUpdateRecordsMetadata.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestCreateSchemaIntegration.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestStorageSchemaNegative.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoBuilder.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoClient.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSTestUtils.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/SchemaUtil.java
delete mode 100644 testing/storage-test-aws-mongodb/src/test/resources/logback-test.xml
diff --git a/.fossa.yml b/.fossa.yml
index 60d2e99d7..2045a8cff 100644
--- a/.fossa.yml
+++ b/.fossa.yml
@@ -32,10 +32,6 @@ analyze:
type: mvn
target: provider/storage-aws/pom.xml
path: .
- - name: storage-aws-mongodb
- type: mvn
- target: provider/storage-aws-mongodb/pom.xml
- path: .
- name: storage-ibm
type: mvn
target: provider/storage-ibm/pom.xml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a1d4b5fc3..b77576c45 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,6 +6,11 @@ variables:
AWS_DEPLOY_TARGET: EKS
AWS_EKS_DEPLOYMENT_NAME: os-storage
+ MONGODB_AWS_SERVICE: storage
+ MONGODB_AWS_BUILD_SUBDIR: provider/storage-aws/build-aws
+ MONGODB_AWS_TEST_SUBDIR: testing/storage-test-aws
+ MONGODB_AWS_EKS_DEPLOYMENT_NAME: os-storage
+
GCP_BUILD_SUBDIR: provider/storage-gcp
GCP_INT_TEST_SUBDIR: testing/storage-test-gcp
GCP_APPLICATION_NAME: os-storage
@@ -35,6 +40,12 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-maven.yml"
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/aws-mongodb-global.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/aws-mongodb-maven.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm.yml"
diff --git a/NOTICE b/NOTICE
index f24a1f8d3..59e12674a 100644
--- a/NOTICE
+++ b/NOTICE
@@ -303,6 +303,7 @@ The following software have components provided under the terms of this license:
- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/, https://commons.apache.org/proper/commons-codec/)
- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/, https://commons.apache.org/proper/commons-collections/)
+- Apache Commons Compress (from http://commons.apache.org/compress/, http://commons.apache.org/proper/commons-compress/, https://commons.apache.org/proper/commons-compress/)
- Apache Commons IO (from http://commons.apache.org/io/, https://commons.apache.org/proper/commons-io/, https://repo1.maven.org/maven2/commons-io/commons-io)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/, https://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/)
@@ -339,6 +340,8 @@ The following software have components provided under the terms of this license:
- Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
- Expression Language 3.0 (from http://el-spec.java.net, http://uel.java.net, https://projects.eclipse.org/projects/ee4j.el)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- Flapdoodle Embedded MongoDB (from http://github.com/flapdoodle-oss/de.flapdoodle.embed.mongo)
+- Flapdoodle Embedded Process Util (from http://github.com/flapdoodle-oss/de.flapdoodle.embed.process)
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine)
@@ -861,6 +864,7 @@ Info-ZIP
========================================================================
The following software have components provided under the terms of this license:
+- Apache Commons Compress (from http://commons.apache.org/compress/, http://commons.apache.org/proper/commons-compress/, https://commons.apache.org/proper/commons-compress/)
- ClassGraph (from https://github.com/classgraph/classgraph)
========================================================================
diff --git a/pom.xml b/pom.xml
index 959db4e6f..b25811f03 100644
--- a/pom.xml
+++ b/pom.xml
@@ -173,7 +173,6 @@
<module>provider/storage-byoc</module>
<module>provider/storage-azure</module>
<module>provider/storage-aws</module>
- <module>provider/storage-aws-mongodb</module>
<module>provider/storage-ibm</module>
<module>provider/storage-reference</module>
</modules>
diff --git a/provider/storage-aws-mongodb/build-aws/Dockerfile b/provider/storage-aws-mongodb/build-aws/Dockerfile
deleted file mode 100644
index 276a36b93..000000000
--- a/provider/storage-aws-mongodb/build-aws/Dockerfile
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
-FROM amazoncorretto:8
-
-ARG JAR_FILE=provider/storage-aws-mongodb/target/*spring-boot.jar
-WORKDIR /
-COPY ${JAR_FILE} app.jar
-EXPOSE 8080
-ENTRYPOINT java $JAVA_OPTS -jar /app.jar
diff --git a/provider/storage-aws-mongodb/build-aws/build-info.py b/provider/storage-aws-mongodb/build-aws/build-info.py
deleted file mode 100644
index 12935424a..000000000
--- a/provider/storage-aws-mongodb/build-aws/build-info.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import boto3
-import json
-import os
-import argparse
-
-# Create the build-info.json
-parser = argparse.ArgumentParser(description="")
-
-# env - CODEBUILD_SOURCE_VERSION
-parser.add_argument("--branch", type=str, help="")
-
-# env - CODEBUILD_RESOLVED_SOURCE_VERSION
-parser.add_argument("--commit", type=str, help="")
-
-# env - CODEBUILD_BUILD_ID
-parser.add_argument("--buildid", type=str, help="")
-
-# env - CODEBUILD_BUILD_NUMBER
-parser.add_argument("--buildnumber", type=str, help="")
-
-# Get from directory name
-parser.add_argument("--reponame", type=str, help="")
-
-# env OUTPUT_DIR
-parser.add_argument("--outdir", type=str, help="")
-
-# full ecr image and tag, and any other artifacts
-parser.add_argument("--artifact", type=str, action="append", help="")
-
-
-
-args = parser.parse_args()
-
-branch = args.branch
-commitId = args.commit
-buildId = args.buildid
-buildNumber = args.buildnumber
-repoName = args.reponame
-outputDir = args.outdir
-artifacts = args.artifact
-
-buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
-
-print(buildInfoFilePath)
-
-commitArgs = {
- "repositoryName": repoName,
- "commitId": commitId
-}
-
-commitDetail = {
- "commit": ""
-}
-
-# get the commit detail
-try:
- codecommit = boto3.client("codecommit")
- commitDetail = codecommit.get_commit(**commitArgs)
-except Exception as e:
- print("Getting commit information from codecommit failed")
-
-buildInfo = {
- "branch": branch,
- "build-id": buildId,
- "build-number": buildNumber,
- "repo": repoName,
- "artifacts": artifacts,
- "commit": commitDetail["commit"]
-}
-print(json.dumps(buildInfo, sort_keys=True, indent=4))
-
-# write the build.json file to dist
-f = open(buildInfoFilePath, "w")
-f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
-f.close()
diff --git a/provider/storage-aws-mongodb/build-aws/buildspec.yaml b/provider/storage-aws-mongodb/build-aws/buildspec.yaml
deleted file mode 100644
index 7a767b3f2..000000000
--- a/provider/storage-aws-mongodb/build-aws/buildspec.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-# Copyright © 2020 Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
-
-# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
-version: 0.2
-
-env:
- secrets-manager:
- DOCKER_USERNAME: /osdu/devops/docker_credentials:username
- DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
-
-phases:
- install:
- runtime-versions:
- java: corretto8
- commands:
- - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - mvn -version
- - echo $JAVA_HOME #WHY
- - mkdir -p /root/.m2
- - cp ./provider/storage-aws-mongodb/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- - export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- - export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain $AWS_OSDU_DEV_MAVEN_DOMAIN --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text`
- pre_build:
- commands:
- - echo "Logging in to Amazon ECR..."
- - $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
- build:
- commands:
- - export REPO_NAME=${PWD##*/}
- - export OUTPUT_DIR="dist"
- - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- - export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- - export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- - export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- - export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
- - export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin
- - mkdir -p ${OUTPUT_DIR}/bin
- - mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- - echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- - printenv
-
- - echo "Building primary service assemblies..."
- - mvn -B test install -pl storage-core,provider/storage-aws-mongodb -Ddeployment.environment=prod
-
- # Suspended until further notice
- # - echo "Copying assemblies to dist..."
- # - cp ./provider/storage-aws-mongodb/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars
- # - cp ./storage-core/target/*.jar ${OUTPUT_DIR}/bin # copy core jar
-
- - echo "Building integration testing assemblies and gathering artifacts..."
- - chmod +x ./testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh
- - ./testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh
-
- - echo "Logging into Docker Hub..."
- - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
-
- - echo "Building docker image..."
- - docker build -f provider/storage-aws-mongodb/build-aws/Dockerfile -t ${ECR_IMAGE} .
- - docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- - echo "Pushing docker image..."
- - docker push ${ECR_IMAGE}
- - docker push ${ECR_IMAGE_BRANCH_LATEST}
-
- - echo "Generate build-info.json"
- - |
- python provider/storage-aws-mongodb/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
- --buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
- --artifact ${ECR_IMAGE}
-reports:
- SurefireReports: # CodeBuild will create a report group called "SurefireReports".
- files: #Store all of the files
- - "storage-core/target/surefire-reports/**/*"
- - "provider/storage-aws-mongodb/target/surefire-reports/**/*"
- base-directory: "." # Location of the reports
-artifacts:
- files:
- - "**/*"
- base-directory: "dist"
- name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
-cache:
- paths:
- - "/root/.m2/**/*"
\ No newline at end of file
diff --git a/provider/storage-aws-mongodb/lombok.config b/provider/storage-aws-mongodb/lombok.config
deleted file mode 100644
index 854efc627..000000000
--- a/provider/storage-aws-mongodb/lombok.config
+++ /dev/null
@@ -1,3 +0,0 @@
-lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier
-lombok.copyableAnnotations += org.springframework.context.annotation.Lazy
-lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Value
diff --git a/provider/storage-aws-mongodb/maven/settings.xml b/provider/storage-aws-mongodb/maven/settings.xml
deleted file mode 100644
index 3dbde15f4..000000000
--- a/provider/storage-aws-mongodb/maven/settings.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Copyright © 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
-
- <profiles>
- <profile>
- <id>aws-osdu-dev-maven</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <repositories>
- <repository>
- <id>aws-osdu-dev-maven</id>
- <url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
- </repository>
- <repository>
- <id>gitlab-os-core-common-maven</id>
- <url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
- </repository>
- <repository>
- <id>gitlab-os-core-lib-aws-maven</id>
- <url>https://community.opengroup.org/api/v4/projects/68/packages/maven</url>
- </repository>
- </repositories>
- </profile>
- <profile>
- <id>credentialsConfiguration</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <deployment.environment>dev</deployment.environment>
- <aws.accessKeyId>no-default</aws.accessKeyId>
- <aws.secretKey>no-default</aws.secretKey>
- <azure.devops.username>Another-Access-Token-2021</azure.devops.username>
- <azure.devops.token>no-default</azure.devops.token>
- </properties>
- </profile>
- </profiles>
-
- <servers>
- <server>
- <id>aws-osdu-dev-maven</id>
- <username>aws</username>
- <password>${env.AWS_OSDU_DEV_MAVEN_AUTH_TOKEN}</password>
- </server>
- </servers>
-
- <!-- CodeArtifact doesn't support external repos yet that aren't Maven Central. ETA Q4 2020. -->
- <!-- <mirrors> -->
- <!-- <mirror> -->
- <!-- <id>aws-osdu-dev-maven</id> -->
- <!-- <name>aws-osdu-dev-maven</name> -->
- <!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
- <!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
- <!-- </mirror> -->
- <!-- </mirrors> -->
-
- <activeProfiles>
- <activeProfile>credentialsConfiguration</activeProfile>
- </activeProfiles>
-
-</settings>
\ No newline at end of file
diff --git a/provider/storage-aws-mongodb/pom.xml b/provider/storage-aws-mongodb/pom.xml
deleted file mode 100644
index 6e6a506e6..000000000
--- a/provider/storage-aws-mongodb/pom.xml
+++ /dev/null
@@ -1,170 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Copyright © 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <parent>
- <artifactId>os-storage</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- <version>0.16.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>storage-aws-mongodb</artifactId>
- <description>Storage service on AWS using MongoDB</description>
- <version>0.16.0-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <properties>
- <aws.version>1.11.637</aws.version>
- <deployment.environment>dev</deployment.environment>
- <version.number>0.16.0-SNAPSHOT</version.number>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bom</artifactId>
- <version>${aws.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <dependencies>
- <!-- Internal packages -->
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu.core.aws</groupId>
- <artifactId>os-core-lib-aws</artifactId>
- <version>0.3.8</version>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>storage-core</artifactId>
- <version>${version.number}</version>
- </dependency>
- <!-- AWS managed packages -->
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk</artifactId>
- <version>${aws.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-dynamodb</artifactId>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sns</artifactId>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-cognitoidentity</artifactId>
- </dependency>
-
- <!-- Third party Apache 2.0 license packages -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-security</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-oauth2-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-oauth2-jose</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.data</groupId>
- <artifactId>spring-data-commons</artifactId>
- <version>2.1.10.RELEASE</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>javax.inject</groupId>
- <artifactId>javax.inject</artifactId>
- <version>1</version>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </dependency>
-
- <!-- Testing packages -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.10.19</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>2.0.2</version>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- <configuration>
- <classifier>spring-boot</classifier>
- <mainClass>org.opengroup.osdu.storage.StorageApplication</mainClass>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageImpl.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageImpl.java
deleted file mode 100644
index 177b4e9de..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageImpl.java
+++ /dev/null
@@ -1,293 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import com.google.gson.Gson;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.storage.RecordData;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-import org.opengroup.osdu.core.common.model.storage.TransferInfo;
-import org.opengroup.osdu.core.common.util.Crc32c;
-import org.opengroup.osdu.storage.provider.mongodb.security.UserAccessService;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.CallableResult;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.RecordProcessor;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.RecordsUtil;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.S3RecordClient;
-import org.opengroup.osdu.storage.provider.interfaces.ICloudStorage;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Repository;
-
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.stream.Collectors;
-
-import static org.apache.commons.codec.binary.Base64.encodeBase64;
-
-@Repository
-public class CloudStorageImpl implements ICloudStorage {
-
- @Value("${aws.dynamodb.table.prefix}")
- String tablePrefix;
-
- @Value("${aws.region}")
- String dynamoDbRegion;
-
- @Value("${aws.dynamodb.endpoint}")
- String dynamoDbEndpoint;
-
- @Value("${aws.s3.max-record-threads}")
- private int maxNumOfRecordThreads;
-
- @Inject
- private S3RecordClient s3RecordClient;
-
- @Inject
- private JaxRsDpsLog logger;
-
- @Inject
- private RecordsUtil recordsUtil;
-
- @Inject
- private UserAccessService userAccessService;
-
- @Inject
- private IRecordsMetadataRepository recordsMetadataRepository;
-
- private ExecutorService threadPool;
-
- private DynamoDBQueryHelper queryHelper;
-
- @PostConstruct
- public void init(){
- this.threadPool = Executors.newFixedThreadPool(maxNumOfRecordThreads);
- queryHelper = new DynamoDBQueryHelper(dynamoDbEndpoint, dynamoDbRegion, tablePrefix);
- }
-
- // Used specifically in the unit tests
- public void init(ExecutorService threadPool){
- this.threadPool = threadPool;
- }
-
- @Override
- public void write(RecordProcessing... recordsProcessing) {
- userAccessService.validateRecordAcl(this.queryHelper, recordsProcessing);
-
- // TODO: throughout this class userId isn't used, seems to be something to integrate with entitlements service
- // TODO: ensure that the threads come from the shared pool manager from the web server
- // Using threads to write records to S3 to increase efficiency, no impact to cost
- List<CompletableFuture<RecordProcessor>> futures = new ArrayList<>();
-
- for(RecordProcessing recordProcessing : recordsProcessing){
- if (recordProcessing.getRecordData().getMeta() == null) {
- HashMap<String, Object> meta = new HashMap<String, Object>();
- HashMap<String, Object>[] arrayMeta = new HashMap[0];
- recordProcessing.getRecordData().setMeta(arrayMeta);
- }
- RecordProcessor recordProcessor = new RecordProcessor(recordProcessing, s3RecordClient);
- CompletableFuture<RecordProcessor> future = CompletableFuture.supplyAsync(recordProcessor::call);
- futures.add(future);
- }
-
- try {
- CompletableFuture[] cfs = futures.toArray(new CompletableFuture[0]);
- CompletableFuture<List<RecordProcessor>> results = CompletableFuture.allOf(cfs)
- .thenApply(ignored -> futures.stream()
- .map(CompletableFuture::join)
- .collect(Collectors.toList()));
-
- List<RecordProcessor> recordProcessors = results.get();
- for(RecordProcessor recordProcessor : recordProcessors){
- if(recordProcessor.exception != null
- || recordProcessor.result == CallableResult.Fail){
- assert recordProcessor.exception != null;
- logger.error(String.format("%s failed writing to S3 with exception: %s"
- , recordProcessor.recordId
- , recordProcessor.exception.getErrorMessage()
- ));
- }
- }
- } catch (Exception e) {
- if (e.getCause() instanceof AppException) {
- throw (AppException) e.getCause();
- } else {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion",
- e.getMessage(), e);
- }
- }
- }
-
- @Override
- public Map<String, String> getHash(Collection<RecordMetadata> records) {
- Collection<RecordMetadata> accessibleRecords = new ArrayList<>();
-
- for (RecordMetadata record : records) {
- if (userAccessService.userHasAccessToRecord(record.getAcl())) {
- accessibleRecords.add(record);
- }
- }
-
- Gson gson = new Gson();
- Map<String, String> base64Hashes = new HashMap<String, String>();
- Map<String, String> recordsMap = recordsUtil.getRecordsValuesById(accessibleRecords);
- for (Map.Entry<String, String> recordObj : recordsMap.entrySet()) {
- String recordId = recordObj.getKey();
- String contents = recordObj.getValue();
- RecordData data = gson.fromJson(contents, RecordData.class);
- String dataContents = gson.toJson(data);
- byte[] bytes = dataContents.getBytes(StandardCharsets.UTF_8);
- Crc32c checksumGenerator = new Crc32c();
- checksumGenerator.update(bytes, 0, bytes.length);
- bytes = checksumGenerator.getValueAsBytes();
- String newHash = new String(encodeBase64(bytes));
- base64Hashes.put(recordId, newHash);
- }
- return base64Hashes;
- }
-
- @Override
- public void delete(RecordMetadata record) {
- if (!record.hasVersion()) {
- this.logger.warning(String.format("Record %s does not have versions available", record.getId()));
- return;
- }
-
- if(userAccessService.userHasAccessToRecord(record.getAcl())) {
- s3RecordClient.deleteRecord(record);
- } else {
- logger.error(String.format("User not in ACL for record %s", record.getId()));
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
- "The user is not authorized to perform this action");
- }
- }
-
- @Override
- public void deleteVersion(RecordMetadata record, Long version) {
- if(userAccessService.userHasAccessToRecord(record.getAcl())) {
- s3RecordClient.deleteRecordVersion(record, version);
- } else {
- logger.error(String.format("User not in ACL for record %s", record.getId()));
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
- "The user is not authorized to perform this action");
- }
- }
-
- @Override
- public boolean hasAccess(RecordMetadata... records) {
- for (RecordMetadata recordMetadata : records) {
- if (!recordMetadata.hasVersion()) {
- this.logger.warning(String.format("Record %s does not have versions available", recordMetadata.getId()));
- continue;
- }
-
- if (!userAccessService.userHasAccessToRecord(recordMetadata.getAcl())) {
- return false;
- }
- }
-
- return true;
- }
-
- @Override
- public String read(RecordMetadata record, Long version, boolean checkDataInconsistency) {
- // checkDataInconsistency not used in other providers
- if(userAccessService.userHasAccessToRecord(record.getAcl())) {
- return s3RecordClient.getRecord(record, version);
- } else {
- logger.error(String.format("User not in ACL for record %s", record.getId()));
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
- "The user is not authorized to perform this action");
- }
- }
-
- @Override
- public Map<String, String> read(Map<String, String> objects) {
- // key -> record id
- // value -> record version path
- return recordsUtil.getRecordsValuesById(objects);
- }
-
- @Override
- public boolean isDuplicateRecord(TransferInfo transfer, Map<String, String> hashMap, Map.Entry<RecordMetadata, RecordData> kv) {
- RecordMetadata metadata = kv.getKey();
- RecordData recordData = kv.getValue();
-
- Gson gson = new Gson();
- String dataContents = gson.toJson(recordData.getData());
- String originalDataContents = s3RecordClient.getRecord(metadata, metadata.getLatestVersion());
- RecordData originalRecordData = gson.fromJson(originalDataContents, RecordData.class);
- originalDataContents = gson.toJson(originalRecordData.getData());
- String newHash = Base64.getEncoder().encodeToString(dataContents.getBytes());
- String originalHash = Base64.getEncoder().encodeToString(originalDataContents.getBytes());
-
- if (newHash.equals(originalHash)) {
- transfer.getSkippedRecords().add(metadata.getId());
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public Map<String, Acl> updateObjectMetadata(List<RecordMetadata> recordsMetadata, List<String> recordsId, List<RecordMetadata> validMetadata, List<String> lockedRecords, Map<String, String> recordsIdMap) {
-
- Map<String, Acl> originalAcls = new HashMap<>();
- Map<String, RecordMetadata> currentRecords = this.recordsMetadataRepository.get(recordsId);
-
- for (RecordMetadata recordMetadata : recordsMetadata) {
- String id = recordMetadata.getId();
- String idWithVersion = recordsIdMap.get(id);
-
- if (!id.equalsIgnoreCase(idWithVersion)) {
- long previousVersion = Long.parseLong(idWithVersion.split(":")[3]);
- long currentVersion = currentRecords.get(id).getLatestVersion();
- if (previousVersion != currentVersion) {
- lockedRecords.add(idWithVersion);
- continue;
- }
- }
- validMetadata.add(recordMetadata);
- originalAcls.put(recordMetadata.getId(), currentRecords.get(id).getAcl());
- }
- return originalAcls;
- }
-
- @Override
- public void revertObjectMetadata(List<RecordMetadata> recordsMetadata, Map<String, Acl> originalAcls) {
- List<RecordMetadata> originalAclRecords = new ArrayList<>();
- for (RecordMetadata recordMetadata : recordsMetadata) {
- Acl acl = originalAcls.get(recordMetadata.getId());
- recordMetadata.setAcl(acl);
- originalAclRecords.add(recordMetadata);
- }
- try {
- this.recordsMetadataRepository.createOrUpdate(originalAclRecords);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error while reverting metadata: in revertObjectMetadata.","Internal server error.", e);
- }
- }
-
-
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImpl.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImpl.java
deleted file mode 100644
index 6f4161c32..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImpl.java
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import com.amazonaws.services.sns.AmazonSNS;
-import com.amazonaws.services.sns.model.MessageAttributeValue;
-import com.amazonaws.services.sns.model.PublishRequest;
-import com.google.gson.Gson;
-import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
-import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
-import org.opengroup.osdu.core.aws.ssm.SSMConfig;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-@Component
-public class MessageBusImpl implements IMessageBus {
-
- private String amazonSNSTopic;
-
- @Value("${aws.region}")
- private String amazonSNSRegion;
-
- @Value("${aws.sns.topic.arn}")
- private String parameter;
-
- private AmazonSNS snsClient;
-
- @Inject
- private JaxRsDpsLog logger;
-
- private ParameterStorePropertySource ssm;
-
- @PostConstruct
- public void init(){
- AmazonSNSConfig config = new AmazonSNSConfig(amazonSNSRegion);
- snsClient = config.AmazonSNS();
- SSMConfig ssmConfig = new SSMConfig();
- ssm = ssmConfig.amazonSSM();
- amazonSNSTopic = ssm.getProperty(parameter).toString();
- }
-
- @Override
- public void publishMessage(DpsHeaders headers, PubSubInfo... messages) {
- final int BATCH_SIZE = 50;
- Gson gson = new Gson();
- for (int i =0; i < messages.length; i+= BATCH_SIZE){
-
- PubSubInfo[] batch = Arrays.copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE));
- String json = gson.toJson(batch);
-
- // Attributes
- Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
- messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
- messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
- headers.addCorrelationIdIfMissing();
- messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getCorrelationId()));
- messageAttributes.put(DpsHeaders.USER_EMAIL, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getUserEmail()));
- messageAttributes.put(DpsHeaders.AUTHORIZATION, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getAuthorization()));
- PublishRequest publishRequest = new PublishRequest(amazonSNSTopic, json)
- .withMessageAttributes(messageAttributes);
-
- logger.info("Storage publishes message " + headers.getCorrelationId());
- snsClient.publish(publishRequest);
-
- }
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/QueryRepositoryImplMongoDB.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/QueryRepositoryImplMongoDB.java
deleted file mode 100644
index 905f7b763..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/QueryRepositoryImplMongoDB.java
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import org.opengroup.osdu.core.aws.dynamodb.QueryPageResult;
-import org.opengroup.osdu.core.aws.mongodb.MongoDBHelper;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
-import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
-import org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents.RecordMetadataDocMongoDB;
-import org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents.SchemaMongoDBDoc;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
-import org.springframework.stereotype.Repository;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-@Repository
-public class QueryRepositoryImplMongoDB implements IQueryRepository {
-
- private DpsHeaders headers;
- private MongoDBHelper queryHelper;
-
- @Autowired
- public QueryRepositoryImplMongoDB(DpsHeaders headers, MongoDBHelper queryHelper) {
- this.headers = headers;
- this.queryHelper = queryHelper;
- }
-
- @Override
- public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
- // Set the page size or use the default constant
- int numRecords = PAGE_SIZE;
- if (limit != null) {
- numRecords = limit > 0 ? limit : PAGE_SIZE;
- }
-
- DatastoreQueryResult datastoreQueryResult = new DatastoreQueryResult();
-
- Criteria criteria = Criteria.where("dataPartitionId").is(headers.getPartitionId());
- criteria.andOperator(Criteria.where("user").is(headers.getUserEmail()));
- Query legalTagQuery = new Query(criteria);
-
- QueryPageResult<SchemaMongoDBDoc> queryPageResult = queryHelper.queryPage(SchemaMongoDBDoc.class, legalTagQuery, "kind", cursor, numRecords);
-
- List<String> kinds = new ArrayList<>();
- for (SchemaMongoDBDoc schemaDoc : queryPageResult.results) {
- kinds.add(schemaDoc.getKind());
- }
-
- // Set the cursor for the next page, if applicable
- datastoreQueryResult.setCursor(queryPageResult.cursor);
-
- //TODO: maybe sort only if null cursor
- Collections.sort(kinds);
- datastoreQueryResult.setResults(kinds);
- return datastoreQueryResult;
- }
-
- @Override
- public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
- // Set the page size, or use the default constant
- int numRecords = PAGE_SIZE;
- if (limit != null) {
- numRecords = limit > 0 ? limit : PAGE_SIZE;
- }
-
- DatastoreQueryResult dqr = new DatastoreQueryResult();
- List<String> ids = new ArrayList<>();
-
-
- Criteria criteria = Criteria.where("kind").is(kind);
- criteria.andOperator(Criteria.where("status").is("active"));
- Query legalTagQuery = new Query(criteria);
- QueryPageResult<RecordMetadataDocMongoDB> scanPageResults = queryHelper.queryPage(RecordMetadataDocMongoDB.class, legalTagQuery, "id", cursor, numRecords);
-
- dqr.setCursor(scanPageResults.cursor); // set the cursor for the next page, if applicable
- scanPageResults.results.forEach(schemaDoc -> ids.add(schemaDoc.getId())); // extract the Kinds from the SchemaDocs
-
- // Sort the IDs alphabetically and set the results
- Collections.sort(ids);
- dqr.setResults(ids);
- return dqr;
- }
-}
\ No newline at end of file
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/RecordsMetadataRepositoryImplMongoDB.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/RecordsMetadataRepositoryImplMongoDB.java
deleted file mode 100644
index 5431c9614..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/RecordsMetadataRepositoryImplMongoDB.java
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import lombok.RequiredArgsConstructor;
-import lombok.val;
-import org.opengroup.osdu.core.aws.dynamodb.QueryPageResult;
-import org.opengroup.osdu.core.aws.mongodb.MongoDBHelper;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents.LegalTagAssociationDocMongoDB;
-import org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents.RecordMetadataDocMongoDB;
-import org.springframework.data.domain.Sort;
-import org.springframework.data.mongodb.core.index.Index;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
-import org.springframework.stereotype.Repository;
-
-import javax.annotation.PostConstruct;
-import java.util.AbstractMap;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-@RequiredArgsConstructor
-@Repository
-public class RecordsMetadataRepositoryImplMongoDB implements IRecordsMetadataRepository<String> {
-
- private final MongoDBHelper queryHelper;
-
- @PostConstruct
- public void init() {
- queryHelper.ensureIndex(LegalTagAssociationDocMongoDB.class, new Index().on("legalTag", Sort.Direction.ASC));
- //TODO: research performance difference about moving RecordMetadataDocMongoDB indexes out from top level of document
- queryHelper.ensureIndex(RecordMetadataDocMongoDB.class, new Index().on("kind", Sort.Direction.ASC));
- queryHelper.ensureIndex(RecordMetadataDocMongoDB.class, new Index().on("status", Sort.Direction.ASC));
- queryHelper.ensureIndex(RecordMetadataDocMongoDB.class, new Index().on("user", Sort.Direction.ASC));
- }
-
- /**
- * Saves or updates entities in DB
- */
- @Override
- public List<RecordMetadata> createOrUpdate(List<RecordMetadata> recordsMetadata) {
- if (recordsMetadata == null) return null;
- createLegalTagAssociations(recordsMetadata);
-
- val recordDocs = recordsMetadata.stream()
- .map(RecordMetadataDocMongoDB::new)
- .collect(Collectors.toList());
-
- // TODO Performance optimizations here
- recordDocs.forEach(queryHelper::save);
-
- return recordsMetadata;
- }
-
- @Override
- public void delete(String id) {
- RecordMetadata rmd = get(id); // needed for authorization check
- queryHelper.deleteByPrimaryKey(RecordMetadataDocMongoDB.class, id);
- for (String legalTag : rmd.getLegal().getLegaltags()) {
- deleteLegalTagAssociation(id, legalTag);
- }
- }
-
- private List<RecordMetadataDocMongoDB> getByIds(Set<String> ids) {
- Query legalTagQuery = new Query(Criteria.where("id").in(ids));
- return queryHelper.getByQuery(legalTagQuery, RecordMetadataDocMongoDB.class);
- }
-
- @Override
- public RecordMetadata get(String id) {
- RecordMetadataDocMongoDB doc = queryHelper.getById(id, RecordMetadataDocMongoDB.class);
- if (doc == null) {
- return null;
- } else {
- return doc.getMetadata();
- }
- }
-
- @Override
- public Map<String, RecordMetadata> get(List<String> ids) {
- return queryHelper.multiGetByField(ids, "id", RecordMetadataDocMongoDB.class)
- .stream()
- .collect(Collectors.toMap(RecordMetadataDocMongoDB::getId, RecordMetadataDocMongoDB::getMetadata));
- }
-
- @Override
- public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName(
- String legalTagName, int limit, String cursor) {
-
- Query legalTagQuery = new Query(Criteria.where("legalTag").is(legalTagName));
- QueryPageResult<LegalTagAssociationDocMongoDB> result = queryHelper.queryPage(LegalTagAssociationDocMongoDB.class,
- legalTagQuery, "recordIdLegalTag", cursor, limit);
-
- Set<String> associatedRecordIds = new HashSet<>();
- result.results.forEach(doc -> associatedRecordIds.add(doc.getRecordId())); // extract the Kinds from the SchemaDocs
-
- List<RecordMetadataDocMongoDB> docList = getByIds(associatedRecordIds);
- List<RecordMetadata> associatedRecords = docList.stream().map(RecordMetadataDocMongoDB::getMetadata).collect(Collectors.toList());
-
- return new AbstractMap.SimpleEntry<>(result.cursor, associatedRecords);
- }
-
- @Override
- public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegal(String legalTagName, LegalCompliance status, int limit) {
- return null;
- }
-
- private void saveLegalTagAssociation(String recordId, Set<String> legalTags) {
- for (String legalTag : legalTags) {
- LegalTagAssociationDocMongoDB doc = new LegalTagAssociationDocMongoDB();
- doc.setLegalTag(legalTag);
- doc.setRecordId(recordId);
- doc.setRecordIdLegalTag(String.format("%s:%s", recordId, legalTag));
- // TODO Do this async
- queryHelper.save(doc);
- }
- }
-
- private void createLegalTagAssociations(List<RecordMetadata> recordMetadataList) {
- val list = new LinkedList<LegalTagAssociationDocMongoDB>();
- for (val record : recordMetadataList) {
- for (String legalTag : record.getLegal().getLegaltags()) {
- LegalTagAssociationDocMongoDB doc = new LegalTagAssociationDocMongoDB();
- doc.setLegalTag(legalTag);
- doc.setRecordId(record.getId());
- doc.setRecordIdLegalTag(String.format("%s:%s", record.getId(), legalTag));
- queryHelper.save(doc); // TODO remove this sometimes
- }
- }
- }
-
- private void saveMultipleDocs(List<RecordMetadataDocMongoDB> recordDocs) {
- // TODO Do some performance optimizations here
- recordDocs.forEach(queryHelper::save);
- }
-
- private void deleteLegalTagAssociation(String recordId, String legalTag) {
- queryHelper.deleteByPrimaryKey(LegalTagAssociationDocMongoDB.class, String.format("%s:%s", recordId, legalTag));
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SchemaRepositoryImplMongoDB.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SchemaRepositoryImplMongoDB.java
deleted file mode 100644
index c11fe4e19..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SchemaRepositoryImplMongoDB.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import org.opengroup.osdu.core.aws.mongodb.MongoDBHelper;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.common.model.storage.SchemaItem;
-import org.opengroup.osdu.storage.provider.interfaces.ISchemaRepository;
-import org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents.SchemaMongoDBDoc;
-import org.springframework.data.domain.Sort;
-import org.springframework.data.mongodb.core.index.Index;
-import org.springframework.stereotype.Repository;
-
-import javax.annotation.PostConstruct;
-import java.util.Arrays;
-import java.util.List;
-
-@Repository
-public class SchemaRepositoryImplMongoDB implements ISchemaRepository {
-
- private DpsHeaders headers;
- private MongoDBHelper queryHelper;
-
- public SchemaRepositoryImplMongoDB(DpsHeaders headers, MongoDBHelper queryHelper) {
- this.headers = headers;
- this.queryHelper = queryHelper;
- }
-
- @PostConstruct
- public void init() {
- queryHelper.ensureIndex(SchemaMongoDBDoc.class, new Index().on("dataPartitionId", Sort.Direction.ASC));
- queryHelper.ensureIndex(SchemaMongoDBDoc.class, new Index().on("user", Sort.Direction.ASC));
- }
-
- @Override
- public void add(Schema schema, String user) {
- SchemaMongoDBDoc sd = new SchemaMongoDBDoc();
- sd.setKind(schema.getKind());
-
- // Check if a schema with this kind already exists
- if (queryHelper.keyExistsInTable(sd.getKind(), SchemaMongoDBDoc.class)) {
- throw new IllegalArgumentException("Schema " + sd.getKind() + " already exists. Can't create again.");
- }
-
- // Complete the SchemaDoc object and add it to the database
- sd.setExtension(schema.getExt());
- sd.setUser(user);
- sd.setSchemaItems(Arrays.asList(schema.getSchema()));
- sd.setDataPartitionId(headers.getPartitionId());
- queryHelper.save(sd);
- }
-
- @Override
- public Schema get(String kind) {
- SchemaMongoDBDoc sd = queryHelper.getById(kind, SchemaMongoDBDoc.class);
- if (sd == null) {
- return null;
- }
-
- // Create a Schema object and assign the values retrieved from MongoDB
- return mapSchema(sd);
- }
-
- @Override
- public void delete(String kind) {
- queryHelper.deleteByPrimaryKey(SchemaMongoDBDoc.class, kind);
- }
-
- private Schema mapSchema(SchemaMongoDBDoc schemaDoc) {
- Schema newSchema = new Schema();
- newSchema.setKind(schemaDoc.getKind());
- List<SchemaItem> schemaItemList = schemaDoc.getSchemaItems();
- //TODO: check for removing this
- SchemaItem[] schemaItemsArray = schemaItemList.toArray(new SchemaItem[0]); // convert List to Array
- newSchema.setSchema(schemaItemsArray);
- newSchema.setExt(schemaDoc.getExtension());
- return newSchema;
- }
-
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/GroupCache.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/GroupCache.java
deleted file mode 100644
index 676ba1fc6..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/GroupCache.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.util.Crc32c;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class GroupCache extends RedisCache<String, Groups> {
- public GroupCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT) {
- super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), 30, String.class, Groups.class);
- }
-
- public static String getGroupCacheKey(DpsHeaders headers) {
- String key = String.format("entitlement-groups:%s:%s", headers.getPartitionIdWithFallbackToAccountId(),
- headers.getAuthorization());
- return Crc32c.hashToBase64EncodedString(key);
- }
-
- public static String getPartitionGroupsCacheKey(String dataPartitionId) {
- String key = String.format("entitlement-groups:data-partition:%s", dataPartitionId);
- return Crc32c.hashToBase64EncodedString(key);
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java
deleted file mode 100644
index 5661707f2..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.cache;
-
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.MultiTenantCache;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-
-@Component("LegalTagCache")
-public class LegalTagCache implements ICache<String, String> {
-
- @Inject
- private TenantInfo tenant;
-
- private final MultiTenantCache<String> caches;
-
- public LegalTagCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_LEGALTAG_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_LEGALTAG_PORT) {
- this.caches = new MultiTenantCache<>(new RedisCache<>(REDIS_LEGALTAG_HOST,Integer.parseInt(REDIS_LEGALTAG_PORT),
- 60 * 60,
- String.class,
- String.class));
- }
-
- @Override
- public void put(String key, String val) {
- this.partitionCache().put(key, val);
- }
-
- @Override
- public String get(String key) {
- return this.partitionCache().get(key);
- }
-
- @Override
- public void delete(String key) {
- this.partitionCache().delete(key);
- }
-
- @Override
- public void clearAll() {
- this.partitionCache().clearAll();
- }
-
- private ICache<String, String> partitionCache() {
- return this.caches.get(String.format("%s:legalTag", this.tenant));
- }
-
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/SchemaCache.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/SchemaCache.java
deleted file mode 100644
index 0909ff793..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/SchemaCache.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class SchemaCache extends RedisCache<String, Schema> {
-
- public SchemaCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SCHEMA_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_SCHEMA_PORT) {
- super(REDIS_SCHEMA_HOST, Integer.parseInt(REDIS_SCHEMA_PORT), 60 * 60, String.class,
- Schema.class);
- }
-
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/config/MongoDBConfiguration.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/config/MongoDBConfiguration.java
deleted file mode 100644
index 5e0e86797..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/config/MongoDBConfiguration.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.opengroup.osdu.storage.provider.mongodb.config;
-
-import com.mongodb.MongoClient;
-import lombok.AllArgsConstructor;
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.core.aws.mongodb.MongoDBHelper;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.data.mongodb.core.MongoTemplate;
-
-import javax.annotation.Nonnull;
-
-@AllArgsConstructor
-@NoArgsConstructor
-@ComponentScan("org.opengroup.osdu.storage.provider")
-@Configuration
-public class MongoDBConfiguration { //extends AbstractMongoConfiguration
-
- @Value("${aws.mongodb.database.name}")
- private String databaseName;
- @Value("${aws.mongodb.connection.url}")
- private String connectionString;
-
- @Bean
- public MongoDBHelper queryHelper() {
- return new MongoDBHelper(connectionString, databaseName);
- }
-
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java
deleted file mode 100644
index 1e8e68d90..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/jobs/LegalComplianceChangeServiceAWSImpl.java
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.jobs;
-
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.legal.jobs.*;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordState;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.mongodb.cache.LegalTagCache;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.util.AbstractMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.singletonList;
-
-@NoArgsConstructor
-@Service
-public class LegalComplianceChangeServiceAWSImpl implements ILegalComplianceChangeService {
-
- private final static String incompliantName = "incompliant";
- private final static String compliantName = "compliant";
-
- @Autowired
- private IRecordsMetadataRepository recordsMetadataRepository;
-
- @Autowired
- private IMessageBus storageMessageBus;
-
- @Autowired
- private StorageAuditLogger auditLogger;
-
- @Autowired
- private JaxRsDpsLog logger;
-
- @Autowired
- private LegalTagCache legalTagCache;
-
- @Override
- public Map<String, LegalCompliance> updateComplianceOnRecords(LegalTagChangedCollection legalTagsChanged,
- DpsHeaders headers) throws ComplianceUpdateStoppedException {
- Map<String, LegalCompliance> output = new HashMap<>();
-
- // TODO: optimize to not have while loop inside a for each
- // We should only get one legal tag change from the queue, the model should
- // reflect that
- for (LegalTagChanged lt : legalTagsChanged.getStatusChangedTags()) {
-
- ComplianceChangeInfo complianceChangeInfo = this.getComplianceChangeInfo(lt);
- if (complianceChangeInfo == null) {
- continue;
- }
-
- AbstractMap.SimpleEntry<String, List<RecordMetadata>> results;
- String cursor = null;
- do {
- results = this.recordsMetadataRepository
- .queryByLegalTagName(lt.getChangedTagName(), 500, cursor);
- cursor = results.getKey();
- List<RecordMetadata> recordsMetadata = results.getValue();
- PubSubInfo[] pubsubInfos = this.updateComplianceStatus(complianceChangeInfo, recordsMetadata, output);
-
- if (lt.getChangedTagStatus() == incompliantName){
- for(RecordMetadata rmd : recordsMetadata){
- this.recordsMetadataRepository.delete(rmd.getId());
- }
- } else {
- this.recordsMetadataRepository.createOrUpdate(recordsMetadata);
- }
-
- StringBuilder recordsId = new StringBuilder();
- for (RecordMetadata recordMetadata : recordsMetadata) {
- recordsId.append(", ").append(recordMetadata.getId());
- }
- this.auditLogger.updateRecordsComplianceStateSuccess(
- singletonList("[" + recordsId.toString().substring(2) + "]"));
-
- this.storageMessageBus.publishMessage(headers, pubsubInfos);
- } while (cursor != null);
- }
-
- return output;
- }
-
- private PubSubInfo[] updateComplianceStatus(ComplianceChangeInfo complianceChangeInfo,
- List<RecordMetadata> recordMetadata, Map<String, LegalCompliance> output) {
- PubSubInfo[] pubsubInfo = new PubSubInfo[recordMetadata.size()];
-
- int i = 0;
- for (RecordMetadata rm : recordMetadata) {
- rm.getLegal().setStatus(complianceChangeInfo.getNewState());
- rm.setStatus(complianceChangeInfo.getNewRecordState());
- pubsubInfo[i] = new PubSubInfo(rm.getId(), rm.getKind(), complianceChangeInfo.getPubSubEvent());
- output.put(rm.getId(), complianceChangeInfo.getNewState());
- i++;
- }
-
- return pubsubInfo;
- }
-
- private ComplianceChangeInfo getComplianceChangeInfo(LegalTagChanged lt) {
- ComplianceChangeInfo output = null;
-
- if (lt.getChangedTagStatus().equalsIgnoreCase(compliantName)) {
- output = new ComplianceChangeInfo(LegalCompliance.compliant, OperationType.update, RecordState.active);
- } else if (lt.getChangedTagStatus().equalsIgnoreCase(incompliantName)) {
- this.legalTagCache.delete(lt.getChangedTagName());
- output = new ComplianceChangeInfo(LegalCompliance.incompliant, OperationType.delete, RecordState.deleted);
- } else {
- this.logger.warning(String.format("Unknown LegalTag compliance status received %s %s",
- lt.getChangedTagStatus(), lt.getChangedTagName()));
- }
-
- return output;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/BasicAuthSecurityConfig.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/BasicAuthSecurityConfig.java
deleted file mode 100644
index ad9955e29..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/BasicAuthSecurityConfig.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.security;
-
-import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
-import org.springframework.security.config.annotation.web.builders.HttpSecurity;
-import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
-
-@EnableWebSecurity
-@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
- @Override
- protected void configure(HttpSecurity http) throws Exception {
- http
- .httpBasic().disable()
- .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/UserAccessService.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/UserAccessService.java
deleted file mode 100644
index 940a35734..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/UserAccessService.java
+++ /dev/null
@@ -1,153 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.security;
-
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
-import org.opengroup.osdu.core.aws.entitlements.GroupsUtil;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.entitlements.EntitlementsException;
-import org.opengroup.osdu.core.common.model.entitlements.GroupInfo;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-import org.opengroup.osdu.storage.provider.mongodb.util.CacheHelper;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-
-@Service
-public class UserAccessService {
-
- private CacheHelper cacheHelper;
- @Inject
- private ICache<String, Groups> cache;
- @Inject
- private IEntitlementsFactory entitlementsFactory;
- @Inject
- private DpsHeaders dpsHeaders;
-
- private static final String ACCESS_DENIED_REASON = "Access denied";
- private static final String ACCESS_DENIED_MSG = "The user is not authorized to perform this action";
-
- @PostConstruct
- public void init() {
- cacheHelper = new CacheHelper();
- }
-
- /**
- * Unideal way to check if user has access to record because a list is being compared
- * for a match in a list. Future improvements include redesigning our dynamo schema to
- * get around this and redesigning dynamo schema to stop parsing the acl out of
- * recordmetadata
- *
- * @param acl
- * @return
- */
- // TODO: Optimize entitlements record ACL design to not compare list against list
- public boolean userHasAccessToRecord(Acl acl) {
- Groups groups = getGroups();
- HashSet<String> allowedGroups = new HashSet<>();
-
- for (String owner : acl.getOwners()) {
- allowedGroups.add(owner);
- }
-
- for (String viewer : acl.getViewers()) {
- allowedGroups.add(viewer);
- }
-
- List<GroupInfo> memberGroups = groups.getGroups();
- HashSet<String> memberGroupsSet = new HashSet<>();
-
- for (GroupInfo memberGroup : memberGroups) {
- memberGroupsSet.add(memberGroup.getEmail());
- }
-
- return allowedGroups.stream().anyMatch(memberGroupsSet::contains);
- }
-
- public void validateRecordAcl (DynamoDBQueryHelper queryHelper, RecordProcessing... records){
- List<String> groupNames = getPartitionGroups(queryHelper).getGroupNames();
- for (RecordProcessing record : records) {
- for (String acl : Acl.flattenAcl(record.getRecordMetadata().getAcl())) {
- String groupName = acl.split("@")[0].toLowerCase();
- if (!groupNames.contains(groupName)) {
- throw new AppException(
- HttpStatus.SC_FORBIDDEN,
- "Invalid ACL",
- String.format("ACL has invalid Group %s", acl));
- }
- }
- }
- }
-
- // TODO: duplicate logic resides in EntitlementsAndCacheServiceImpl
- private Groups getGroups() {
- String cacheKey = this.cacheHelper.getGroupCacheKey(this.dpsHeaders);
- Groups groups = this.cache.get(cacheKey);
-
- if (groups == null) {
- groups = refreshGroups();
- }
-
- return groups;
- }
-
- private Groups refreshGroups() {
- Groups groups;
- IEntitlementsService service = this.entitlementsFactory.create(this.dpsHeaders);
-
- try {
- groups = service.getGroups();
- this.cache.put(this.cacheHelper.getGroupCacheKey(this.dpsHeaders), groups);
- } catch (EntitlementsException e) {
- throw new AppException(e.getHttpResponse().getResponseCode(), ACCESS_DENIED_REASON, ACCESS_DENIED_MSG, e);
- }
-
- return groups;
- }
-
- private Groups getPartitionGroups(DynamoDBQueryHelper queryHelper) {
- Groups groups = this.cache.get(this.cacheHelper.getPartitionGroupsCacheKey(this.dpsHeaders.getPartitionId()));
-
- if (groups == null) {
- groups = refreshPartitionGroups(queryHelper);
- }
-
- return groups;
- }
-
- private Groups refreshPartitionGroups (DynamoDBQueryHelper queryHelper) {
- Groups groups = new Groups();
-
- try{
- groups.setGroups(GroupsUtil.getPartitionGroups(queryHelper, dpsHeaders.getPartitionId()));
- this.cache.put(this.cacheHelper.getPartitionGroupsCacheKey(dpsHeaders.getPartitionId()), groups);
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_REASON, ACCESS_DENIED_MSG, e);
- }
-
- return groups;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/WhoamiController.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/WhoamiController.java
deleted file mode 100644
index c6070c36e..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/security/WhoamiController.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.security;
-
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-@Controller
-public class WhoamiController {
- @RequestMapping(value = "/whoami")
- @ResponseBody
- public String whoami() {
- final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
-
- String userName = auth.getName();
- String roles = String.valueOf(auth.getAuthorities());
- String details = String.valueOf(auth.getPrincipal());
-
- return "user: " + userName + "<BR>" +
- "roles: " + roles + "<BR>" +
- "details: " + details + "<BR>";
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/service/BatchServiceAWSImpl.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/service/BatchServiceAWSImpl.java
deleted file mode 100644
index b48b6924f..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/service/BatchServiceAWSImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.service;
-
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.aws.exceptions.InvalidCursorException;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
-import org.opengroup.osdu.storage.service.BatchServiceImpl;
-import org.springframework.stereotype.Service;
-
-import javax.inject.Inject;
-
-import static java.util.Collections.singletonList;
-
-@Service
-public class BatchServiceAWSImpl extends BatchServiceImpl {
- @Inject
- private StorageAuditLogger auditLogger;
-
- @Inject
- private IQueryRepository queryRepository;
-
-// @Inject
-// private JaxRsDpsLog logger;
-
- @Override
- public DatastoreQueryResult getAllKinds(String cursor, Integer limit)
- {
- try {
- DatastoreQueryResult result = this.queryRepository.getAllKinds(limit, cursor);
- this.auditLogger.readAllKindsSuccess(result.getResults());
- return result;
- } catch (InvalidCursorException e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Cursor invalid", "The requested cursor does not exist or is invalid", e);
- } catch (Exception e) {
- throw this.getInternalErrorException(e);
- }
- }
-
- @Override
- public DatastoreQueryResult getAllRecords(String cursor, String kind, Integer limit) {
- try {
- DatastoreQueryResult result = this.queryRepository.getAllRecordIdsFromKind(kind, limit, cursor);
- if (!result.getResults().isEmpty()) {
- this.auditLogger.readAllRecordsOfGivenKindSuccess(singletonList(kind));
- }
- return result;
- } catch (InvalidCursorException e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Cursor invalid", "The requested cursor does not exist or is invalid", e);
- } catch (Exception e) {
- throw this.getInternalErrorException(e);
- }
- }
-
- private AppException getInternalErrorException(Exception e) {
- return new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", e.getMessage());
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/CacheHelper.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/CacheHelper.java
deleted file mode 100644
index dac832722..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/CacheHelper.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.storage.provider.mongodb.cache.GroupCache;
-
-public class CacheHelper {
- public String getGroupCacheKey(DpsHeaders headers){
- return GroupCache.getGroupCacheKey(headers);
- }
-
- public String getPartitionGroupsCacheKey (String dataPartitionId) {
- return GroupCache.getPartitionGroupsCacheKey(dataPartitionId);
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/LegalTagAssociationDocMongoDB.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/LegalTagAssociationDocMongoDB.java
deleted file mode 100644
index 28b02a51c..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/LegalTagAssociationDocMongoDB.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.core.aws.mongodb.BasicMongoDBDoc;
-import org.springframework.data.annotation.Id;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class LegalTagAssociationDocMongoDB implements BasicMongoDBDoc {
-
- @Id
- private String recordIdLegalTag;
-
- private String recordId;
-
- private String legalTag;
-
- @Override
- public String getCursor() {
- return recordIdLegalTag;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/RecordMetadataDocMongoDB.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/RecordMetadataDocMongoDB.java
deleted file mode 100644
index b8897ae2f..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/RecordMetadataDocMongoDB.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.core.aws.mongodb.BasicMongoDBDoc;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.springframework.data.annotation.Id;
-
-import java.util.Set;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class RecordMetadataDocMongoDB implements BasicMongoDBDoc {
-
- @Id
- private String id;
-
- private String kind;
-
- private String status;
-
- private String user;
-
- private RecordMetadata metadata;
-
- private Set<String> legaltags;
-
- public RecordMetadataDocMongoDB(RecordMetadata recordMetadata) {
- id = recordMetadata.getId();
- metadata = recordMetadata;
- kind = recordMetadata.getKind();
- legaltags = recordMetadata.getLegal().getLegaltags();
- status = recordMetadata.getStatus().name();
- user = recordMetadata.getUser();
- }
-
- @Override
- public String getCursor() {
- return id;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/SchemaMongoDBDoc.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/SchemaMongoDBDoc.java
deleted file mode 100644
index 017e78cbf..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/mongodb/documents/SchemaMongoDBDoc.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.opengroup.osdu.storage.provider.mongodb.util.mongodb.documents;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.core.aws.mongodb.BasicMongoDBDoc;
-import org.opengroup.osdu.core.common.model.storage.SchemaItem;
-import org.springframework.data.annotation.Id;
-
-import java.util.List;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class SchemaMongoDBDoc implements BasicMongoDBDoc {
-
- @Id
- private String kind;
-
- private String dataPartitionId;
-
- private String user;
-
- private List<SchemaItem> schemaItems;
-
- private Map<String, Object> extension;
-
- @Override
- public String getCursor() {
- return kind;
- }
-}
-
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordFromVersionTask.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordFromVersionTask.java
deleted file mode 100644
index f249c9a44..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordFromVersionTask.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
-
-import com.amazonaws.AmazonServiceException;
-
-import java.util.concurrent.Callable;
-
-class GetRecordFromVersionTask implements Callable<GetRecordFromVersionTask> {
- private S3RecordClient s3RecordClient;
- private String versionPath;
- public String recordId;
- public String recordContents;
- public Exception exception;
- public CallableResult result;
-
- private final static String EMPTY_S3_MSG = "S3 returned empty record contents";
-
- public GetRecordFromVersionTask(S3RecordClient s3RecordClient,
- String recordId,
- String versionPath){
- this.s3RecordClient = s3RecordClient;
- this.recordId = recordId;
- this.versionPath = versionPath;
- }
-
- @Override
- public GetRecordFromVersionTask call() {
- result = CallableResult.Pass;
- try {
- this.recordContents = s3RecordClient.getRecord(this.versionPath);
-
- if (this.recordContents == null || this.recordContents == ""){
- // s3 wasn't ready to deliver contents
- exception = new Exception(EMPTY_S3_MSG);
- result = CallableResult.Fail;
- }
- }
- catch(AmazonServiceException e) {
- exception = e;
- result = CallableResult.Fail;
- }
- return this;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordTask.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordTask.java
deleted file mode 100644
index 059d7ce6d..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/GetRecordTask.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
-
-import com.amazonaws.AmazonServiceException;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicReference;
-
-class GetRecordTask implements Callable<GetRecordTask> {
- private S3RecordClient s3RecordClient;
- private AtomicReference<Map<String, String>> map;
- public RecordMetadata recordMetadata;
- public AmazonServiceException exception;
- public CallableResult result;
-
- public GetRecordTask(S3RecordClient s3RecordClient,
- AtomicReference<Map<String, String>> map,
- RecordMetadata recordMetadata){
- this.s3RecordClient = s3RecordClient;
- this.map = map;
- this.recordMetadata = recordMetadata;
- }
-
- @Override
- public GetRecordTask call() {
- try{
- s3RecordClient.getRecord(recordMetadata, map);
- result = CallableResult.Pass;
- }
- catch(AmazonServiceException e) {
- result = CallableResult.Fail;
- exception = e;
- }
- return this;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordProcessor.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordProcessor.java
deleted file mode 100644
index bf7dfb1ef..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordProcessor.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
-
-import com.amazonaws.AmazonServiceException;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-
-import java.util.concurrent.Callable;
-
-public class RecordProcessor implements Callable<RecordProcessor> {
- private RecordProcessing recordProcessing;
- private S3RecordClient s3Client;
- public CallableResult result;
- public AmazonServiceException exception;
- public String recordId;
-
- public RecordProcessor(RecordProcessing recordProcessing, S3RecordClient s3Client){
- this.recordProcessing = recordProcessing;
- this.s3Client = s3Client;
- }
-
- @Override
- public RecordProcessor call() {
- try {
- recordId = recordProcessing.getRecordMetadata().getId();
- s3Client.saveRecord(recordProcessing);
- result = CallableResult.Pass;
- }
- catch(AmazonServiceException exception) {
- this.exception = exception;
- result = CallableResult.Fail;
- }
- return this;
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordsUtil.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordsUtil.java
deleted file mode 100644
index 5806eb826..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/RecordsUtil.java
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
-
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-@Component
-public class RecordsUtil {
-
- @Inject
- private JaxRsDpsLog logger;
-
- private S3RecordClient s3RecordClient;
- private ExecutorService threadPool;
-
- public RecordsUtil(S3RecordClient s3RecordClient, ExecutorService threadPool){
- this.s3RecordClient = s3RecordClient;
- this.threadPool = threadPool;
- }
-
- public Map<String, String> getRecordsValuesById(Map<String, String> objects) {
- Map<String, String> map = new HashMap<>();
- List<CompletableFuture<GetRecordFromVersionTask>> futures = new ArrayList<>();
-
- try {
- for (Map.Entry<String, String> object : objects.entrySet()) {
- GetRecordFromVersionTask task = new GetRecordFromVersionTask(s3RecordClient, object.getKey(), object.getValue());
- CompletableFuture<GetRecordFromVersionTask> future = CompletableFuture.supplyAsync(task::call);
- futures.add(future);
- }
-
- CompletableFuture[] cfs = futures.toArray(new CompletableFuture[0]);
- CompletableFuture<List<GetRecordFromVersionTask>> results = CompletableFuture.allOf(cfs)
- .thenApply(ignored -> futures.stream()
- .map(CompletableFuture::join)
- .collect(Collectors.toList()));
-
- List<GetRecordFromVersionTask> getRecordFromVersionTasks = results.get();
- for (GetRecordFromVersionTask task : getRecordFromVersionTasks) {
- if (task.exception != null
- || task.result == CallableResult.Fail) {
- assert task.exception != null;
- logger.error(String.format("%s failed getting record from S3 with exception: %s"
- , task.recordId
- , task.exception.getMessage()
- ));
- } else {
- map.put(task.recordId, task.recordContents);
- }
- }
- } catch (Exception e) {
- if (e.getCause() instanceof AppException) {
- throw (AppException) e.getCause();
- } else {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion",
- e.getMessage(), e);
- }
- }
-
- return map;
- }
-
- public Map<String, String> getRecordsValuesById(Collection<RecordMetadata> recordMetadatas) {
- AtomicReference<Map<String, String>> map = new AtomicReference<>();
- map.set(new HashMap<>());
- List<CompletableFuture<GetRecordTask>> futures = new ArrayList<>();
-
- try {
- for (RecordMetadata recordMetadata: recordMetadatas) {
- GetRecordTask task = new GetRecordTask(s3RecordClient, map, recordMetadata);
- CompletableFuture<GetRecordTask> future = CompletableFuture.supplyAsync(task::call);
- futures.add(future);
- }
-
- CompletableFuture[] cfs = futures.toArray(new CompletableFuture[0]);
- CompletableFuture<List<GetRecordTask>> results = CompletableFuture.allOf(cfs)
- .thenApply(ignored -> futures.stream()
- .map(CompletableFuture::join)
- .collect(Collectors.toList()));
-
- List<GetRecordTask> getRecordTasks = results.get();
- for (GetRecordTask task : getRecordTasks) {
- if (task.exception != null
- || task.result == CallableResult.Fail) {
- logger.error(String.format("%s failed writing to S3 with exception: %s"
- , task.recordMetadata.getId()
- , task.exception.getErrorMessage()
- ));
- }
- }
- } catch (Exception e) {
- if (e.getCause() instanceof AppException) {
- throw (AppException) e.getCause();
- } else {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion",
- e.getMessage(), e);
- }
- }
-
- return map.get();
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/S3RecordClient.java b/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/S3RecordClient.java
deleted file mode 100644
index b297cedc4..000000000
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/S3RecordClient.java
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
-
-import com.amazonaws.SdkClientException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.aws.s3.S3Config;
-import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
-import org.opengroup.osdu.core.aws.ssm.SSMConfig;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.storage.RecordData;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
-
-@Component
-public class S3RecordClient {
-
- @Inject
- private JaxRsDpsLog logger;
-
- // Storing all records in one bucket does not impact performance, no need to spread keys anymore
- private String recordsBucketName;
-
- @Value("${aws.region}")
- private String s3Region;
-
- @Value("${aws.s3.endpoint}")
- private String s3Endpoint;
-
- private AmazonS3 s3;
-
- @Value("${aws.sns.s3.bucket-name}")
- String ssmParameter;
-
- private ParameterStorePropertySource ssm;
-
- @PostConstruct
- public void init() {
- S3Config config = new S3Config(s3Endpoint, s3Region);
- s3 = config.amazonS3();
- SSMConfig ssmConfig = new SSMConfig();
- ssm = ssmConfig.amazonSSM();
- recordsBucketName = ssm.getProperty(ssmParameter).toString();
- }
-
- /**
- * Upload the record to S3
- * This function is call via threads outside of the request scope and so it CANNOT log messages
- * @param recordProcessing
- */
- public void saveRecord(RecordProcessing recordProcessing) {
- Gson gson = new GsonBuilder().serializeNulls().create();
- RecordMetadata recordMetadata = recordProcessing.getRecordMetadata();
- RecordData recordData = recordProcessing.getRecordData();
- String content = gson.toJson(recordData);
- String keyName = getKeyNameForLatestVersion(recordMetadata);
- s3.putObject(recordsBucketName, keyName, content);
- }
-
- public String getRecord(RecordMetadata recordMetadata, Long version) {
- String keyName = getKeyNameForVersion(recordMetadata, version);
- return getRecord(keyName);
- }
-
- public boolean getRecord(RecordMetadata recordMetadata, AtomicReference<Map<String, String>> map) {
- Map<String, String> mapVal = map.get();
- String keyName = getKeyNameForLatestVersion(recordMetadata);
- String record = getRecord(keyName);
- mapVal.put(recordMetadata.getId(), record);
- map.set(mapVal);
- return true;
- }
-
- public boolean getRecord(String recordId, String versionPath, AtomicReference<Map<String, String>> map) {
- Map<String, String> mapVal = map.get();
- String record = getRecord(versionPath);
- mapVal.put(recordId, record);
- map.set(mapVal);
- return true;
- }
-
- public void deleteRecord(RecordMetadata recordMetadata) {
- String keyName = getKeyNameForAllVersions(recordMetadata);
- try {
- s3.deleteObject(new DeleteObjectRequest(recordsBucketName, keyName));
- } catch (SdkClientException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error deleting record", e.getMessage(), e);
- }
- }
-
- public void deleteRecordVersion(RecordMetadata recordMetadata, Long version) {
- String keyName = getKeyNameForVersion(recordMetadata, version);
- try {
- s3.deleteObject(new DeleteObjectRequest(recordsBucketName, keyName));
- } catch (SdkClientException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error deleting record", e.getMessage(), e);
- }
- }
-
- public boolean checkIfRecordExists(RecordMetadata recordMetadata) {
- String keyName = getKeyNameForAllVersions(recordMetadata);
- boolean exists = false;
- try {
- ListObjectsV2Result result = s3.listObjectsV2(recordsBucketName, keyName);
- exists = result.getKeyCount() > 0;
- } catch (SdkClientException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error finding record", e.getMessage(), e);
- }
- return exists;
- }
-
- public String getRecord(String keyName) {
- String record = "";
- try {
- record = s3.getObjectAsString(recordsBucketName, keyName);
- } catch (SdkClientException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error getting record", e.getMessage(), e);
- }
- return record;
- }
-
- private String getKeyNameForLatestVersion(RecordMetadata recordMetadata) {
- return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion();
- }
-
- private String getKeyNameForVersion(RecordMetadata recordMetadata, Long version) {
- return recordMetadata.getVersionPath(version);
- }
-
- private String getKeyNameForAllVersions(RecordMetadata recordMetadata) {
- return recordMetadata.getKind() + "/" + recordMetadata.getId();
- }
-}
diff --git a/provider/storage-aws-mongodb/src/main/resources/application.properties b/provider/storage-aws-mongodb/src/main/resources/application.properties
deleted file mode 100644
index 892ffb7f6..000000000
--- a/provider/storage-aws-mongodb/src/main/resources/application.properties
+++ /dev/null
@@ -1,45 +0,0 @@
-LOG_PREFIX=storage
-
-AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1
-LEGALTAG_API=${LEGALTAG_BASE_URL}/api/legal/v1
-
-CRS_API=notused
-
-server.servlet.contextPath=/api/storage/v2/
-logging.level.org.springframework.web=${LOG_LEVEL}
-server.port=${APPLICATION_PORT}
-
-## AWS DynamoDB configuration
-aws.dynamodb.key=kind
-aws.dynamodb.table.prefix=${ENVIRONMENT}-
-aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
-
-## AWS S3 configuration
-aws.region=${AWS_REGION}
-aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
-aws.s3.max-record-threads=2000
-aws.s3.enable-https=true
-S3_DATA_BUCKET=""
-
-## AWS ElastiCache configuration
-aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
-aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
-
-
-aws.ssm=${SSM_ENABLED}
-aws.parameter.prefix=/osdu/${ENVIRONMENT}
-
-aws.sns.topic.arn=${aws.parameter.prefix}/storage/storage-sns-topic-arn
-aws.sns.s3.bucket-name=${aws.parameter.prefix}/storage/storage-s3-bucket-name
-
-## MongoDB configuration
-aws.mongodb.database.name=osdu_storage_${ENVIRONMENT}
-aws.mongodb.database.cluster=${MONGO_DB_CLUSTER}
-aws.mongodb.database.user=${MONGO_DB_USER}
-aws.mongodb.database.password=${MONGO_DB_PASSWORD}
-aws.mongodb.connection.url=mongodb+srv://${aws.mongodb.database.user}:${aws.mongodb.database.password}@${aws.mongodb.database.cluster}/${aws.mongodb.database.name}?retryWrites=true&w=majority
-
-## Disable Mongodb auto configuration
-spring.autoconfigure.exclude= \
- org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration,\
- org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration
\ No newline at end of file
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageTest.java
deleted file mode 100644
index b1e1774b8..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/CloudStorageTest.java
+++ /dev/null
@@ -1,169 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import com.google.gson.Gson;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.storage.RecordData;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.util.Crc32c;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.provider.mongodb.security.UserAccessService;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.RecordsUtil;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.S3RecordClient;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import javax.inject.Inject;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-import static org.apache.commons.codec.binary.Base64.encodeBase64;
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@RunWith(MockitoJUnitRunner.class)
-@SpringBootTest(classes={StorageApplication.class})
-public class CloudStorageTest {
-
- @InjectMocks
- // Created inline instead of with autowired because mocks were overwritten
- // due to lazy loading
- private CloudStorageImpl repo = new CloudStorageImpl();
-
- @Mock
- private S3RecordClient s3RecordClient;
-
- @Mock
- private RecordsUtil recordsUtil;
-
- @Mock
- private ExecutorService threadPool;
-
- @Mock
- private UserAccessService userAccessService;
-
- @Mock
- private IRecordsMetadataRepository recordsMetadataRepository;
-
- @Inject
- private JaxRsDpsLog logger;
-
- String userId = "test-user-id";
- RecordMetadata record = new RecordMetadata();
- Collection<RecordMetadata> records = new ArrayList<RecordMetadata>();
-
- @Before
- public void setUp() {
- initMocks(this);
- record.setId("test-record-id");
- record.addGcsPath(1);
- records.add(record);
- }
-
-
- @Test
- public void getHash() throws NoSuchFieldException {
- // arrange
- String mockRecord = "{data:{\"id\":\"test\"}}";
- Map<String, String> mapRecords = new HashMap<String, String>();
- mapRecords.put("test-record-id", mockRecord);
- Gson gson = new Gson();
- RecordData data = gson.fromJson(mockRecord, RecordData.class);
- String dataContents = gson.toJson(data);
- byte[] bytes = dataContents.getBytes(StandardCharsets.UTF_8);
- Crc32c checksumGenerator = new Crc32c();
- checksumGenerator.update(bytes, 0, bytes.length);
- bytes = checksumGenerator.getValueAsBytes();
- String expectedHash = new String(encodeBase64(bytes));
-
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.anyObject()))
- .thenReturn(true);
- Mockito.when(recordsUtil.getRecordsValuesById(Mockito.eq(records)))
- .thenReturn(mapRecords);
-
- // act
- Map<String, String> hashMap = repo.getHash(records);
-
- // assert
- Assert.assertEquals(expectedHash, hashMap.get("test-record-id"));
- }
-
- @Test
- public void delete(){
- // arrange
- Mockito.doNothing().when(s3RecordClient).deleteRecord(Mockito.eq(record));
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.anyObject()))
- .thenReturn(true);
-
- // act
- repo.delete(record);
-
- // assert
- Mockito.verify(s3RecordClient, Mockito.times(1)).deleteRecord(record);
- }
-
- @Test
- public void read(){
- // arrange
- Long version = 1L;
- Mockito.when(s3RecordClient.getRecord(Mockito.eq(record), Mockito.eq(version)))
- .thenReturn("test-response");
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.anyObject()))
- .thenReturn(true);
-
- // act
- String resp = repo.read(record, version, false);
-
- // assert
- Assert.assertEquals("test-response", resp);
- }
-
- @Test
- public void readMult(){
- // arrange
- Map<String, String> map = new HashMap<>();
- map.put("test-record-id", "test-version-path");
-
- Map<String, String> expectedResp = new HashMap<>();
- expectedResp.put("test-record-id", "{data:test-data}");
-
- RecordMetadata recordMetadata = new RecordMetadata();
-
- Mockito.when(recordsMetadataRepository.get("test-record-id")).thenReturn(recordMetadata);
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.anyObject()))
- .thenReturn(true);
-
- Mockito.when(recordsUtil.getRecordsValuesById(Mockito.eq(map)))
- .thenReturn(expectedResp);
-
- // act
- Map<String, String> resp = repo.read(map);
-
- // assert
- Assert.assertEquals(expectedResp.get("test-record-id"), resp.get("test-record-id"));
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java
deleted file mode 100644
index fe376ac0e..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/LegalComplianceChangeServiceAWSImplTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentCaptor;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.legal.Legal;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceUpdateStoppedException;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChanged;
-import org.opengroup.osdu.core.common.model.legal.jobs.LegalTagChangedCollection;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.logging.StorageAuditLogger;
-import org.opengroup.osdu.storage.provider.mongodb.cache.LegalTagCache;
-import org.opengroup.osdu.storage.provider.mongodb.jobs.LegalComplianceChangeServiceAWSImpl;
-import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
-import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import java.util.*;
-
-@RunWith(MockitoJUnitRunner.class)
-@SpringBootTest(classes={StorageApplication.class})
-public class LegalComplianceChangeServiceAWSImplTest {
-
- @InjectMocks
- // Created inline instead of with autowired because mocks were overwritten
- // due to lazy loading
- private LegalComplianceChangeServiceAWSImpl service = new LegalComplianceChangeServiceAWSImpl();
-
- @Mock
- private IRecordsMetadataRepository repo;
-
- @Mock
- private IMessageBus storageMessageBus;
-
- @Mock
- private JaxRsDpsLog logger;
-
- @Mock
- private StorageAuditLogger auditLogger;
-
- @Mock
- private LegalTagCache legalTagCache;
-
- @Test
- public void updateComplianceOnRecordsTest() throws ComplianceUpdateStoppedException {
- // arrange
- String incompliantTagName = "incompliant-test-tag";
- String incompliantRecordId = "incompliant-record";
- String compliantTagName = "compliant-test-tag";
- String compliantRecordId = "compliant-record";
-
- // create parameters
- LegalTagChangedCollection legalTagsChanged = new LegalTagChangedCollection();
- List<LegalTagChanged> legalTagChangedList = new ArrayList<>();
- LegalTagChanged incompliantLegalTagChanged = new LegalTagChanged();
- incompliantLegalTagChanged.setChangedTagName(incompliantTagName);
- incompliantLegalTagChanged.setChangedTagStatus("incompliant");
- legalTagChangedList.add(incompliantLegalTagChanged);
- LegalTagChanged compliantLegalTagChanged = new LegalTagChanged();
- compliantLegalTagChanged.setChangedTagName(compliantTagName);
- compliantLegalTagChanged.setChangedTagStatus("compliant");
- legalTagChangedList.add(compliantLegalTagChanged);
- legalTagsChanged.setStatusChangedTags(legalTagChangedList);
-
- DpsHeaders headers = new DpsHeaders();
-
- // incompliant record(s)
- String cursor = null;
- RecordMetadata recordMetadata = new RecordMetadata();
- recordMetadata.setId(incompliantRecordId);
- Legal incompliantLegal = new Legal();
- Set<String> incompliantLegalTags = new HashSet<>();
- incompliantLegalTags.add("incompliant-tag");
- incompliantLegal.setLegaltags(incompliantLegalTags);
- recordMetadata.setLegal(incompliantLegal);
- List<RecordMetadata> incompliantRecordMetaDatas = new ArrayList<>();
- incompliantRecordMetaDatas.add(recordMetadata);
- AbstractMap.SimpleEntry<String, List<RecordMetadata>> incompliantResult =
- new AbstractMap.SimpleEntry<String, List<RecordMetadata>>(cursor, incompliantRecordMetaDatas);
-
- // compliant record(s)
- RecordMetadata compliantRecordMetadata = new RecordMetadata();
- compliantRecordMetadata.setId(compliantRecordId);
- Legal compliantLegal = new Legal();
- Set<String> compliantLegalTags = new HashSet<>();
- compliantLegalTags.add("compliant-tag");
- compliantLegal.setLegaltags(compliantLegalTags);
- compliantRecordMetadata.setLegal(compliantLegal);
- List<RecordMetadata> compliantRecordMetaDatas = new ArrayList<>();
- compliantRecordMetaDatas.add(compliantRecordMetadata);
- AbstractMap.SimpleEntry<String, List<RecordMetadata>> compliantResult =
- new AbstractMap.SimpleEntry<String, List<RecordMetadata>>(cursor, compliantRecordMetaDatas);
-
- // incompliant pub sub info
- PubSubInfo incompliantPubSubInfo = new PubSubInfo();
- incompliantPubSubInfo.setId(incompliantRecordId);
- incompliantPubSubInfo.setOp(OperationType.delete);
- PubSubInfo[] incompliantPubSubInfos = new PubSubInfo[1];
- incompliantPubSubInfos[0] = incompliantPubSubInfo;
-
- // compliant pub sub info
- PubSubInfo compliantPubSubInfo = new PubSubInfo();
- compliantPubSubInfo.setId(compliantRecordId);
- compliantPubSubInfo.setOp(OperationType.update);
- PubSubInfo[] compliantPubSubInfos = new PubSubInfo[1];
- compliantPubSubInfos[0] = compliantPubSubInfo;
-
- // expected output
- Map<String, LegalCompliance> expectedOutput = new HashMap<>();
- expectedOutput.put(incompliantRecordId, LegalCompliance.incompliant);
- expectedOutput.put(compliantRecordId, LegalCompliance.compliant);
-
- // mock methods called
- Mockito.when(repo.queryByLegalTagName(Mockito.eq(incompliantTagName), Mockito.eq(500), Mockito.any()))
- .thenReturn(incompliantResult);
-
- Mockito.when(repo.queryByLegalTagName(Mockito.eq(compliantTagName), Mockito.eq(500), Mockito.any()))
- .thenReturn(compliantResult);
-
- ArgumentCaptor<PubSubInfo[]> pubSubArg = ArgumentCaptor.forClass(PubSubInfo[].class);
-
- // act
- Map<String, LegalCompliance> output = service.updateComplianceOnRecords(legalTagsChanged, headers);
-
- // assert
- // that delete is called on the record returned for incompliant
- Mockito.verify(repo, Mockito.times(1)).delete(incompliantRecordId);
- // that create is called on the record returned for compliant
- Mockito.verify(repo, Mockito.times(1)).createOrUpdate(compliantRecordMetaDatas);
-
- // that storageMessageBus publishMessage is called with the right pubsubinfos
- Mockito.verify(storageMessageBus, Mockito.times(2))
- .publishMessage(Mockito.any(), pubSubArg.capture());
- List<PubSubInfo[]> captured = pubSubArg.getAllValues();
- Object incompliantPubSubObj = captured.get(0);
- PubSubInfo incompliantPubSub = (PubSubInfo) incompliantPubSubObj;
- Object compliantPubSubObj = captured.get(1);
- PubSubInfo compliantPubSub = (PubSubInfo) compliantPubSubObj;
-
- Assert.assertEquals(incompliantPubSubInfos[0].getId(), incompliantPubSub.getId());
- Assert.assertEquals(incompliantPubSubInfos[0].getOp(), incompliantPubSub.getOp());
- Assert.assertEquals(compliantPubSubInfos[0].getId(), compliantPubSub.getId());
- Assert.assertEquals(compliantPubSubInfos[0].getOp(), compliantPubSub.getOp());
-
- // that output returned is expected
- Assert.assertEquals(output.get(incompliantRecordId), expectedOutput.get(incompliantRecordId));
- Assert.assertEquals(output.get(compliantRecordId), expectedOutput.get(compliantRecordId));
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImplTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImplTest.java
deleted file mode 100644
index 2c3c6e793..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/MessageBusImplTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import com.amazonaws.services.sns.AmazonSNS;
-import com.amazonaws.services.sns.model.MessageAttributeValue;
-import com.amazonaws.services.sns.model.PublishRequest;
-import com.amazonaws.services.sns.model.PublishResult;
-import com.google.gson.Gson;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@RunWith(MockitoJUnitRunner.class)
-@SpringBootTest(classes = {StorageApplication.class})
-public class MessageBusImplTest {
-
- @InjectMocks
- private MessageBusImpl messageBus = new MessageBusImpl();
-
- @Mock
- private AmazonSNS snsClient;
-
- @Mock
- private JaxRsDpsLog logger;
-
- @Before
- public void setUp() {
- initMocks(this);
- }
-
- @Test
- public void publishMessage() {
- // arrange
- String amazonSNSTopic = null;
- DpsHeaders headers = new DpsHeaders();
- PubSubInfo message = new PubSubInfo();
- message.setKind("common:welldb:wellbore:1.0.12311");
- message.setOp(OperationType.create_schema);
-
- PubSubInfo[] messages = new PubSubInfo[1];
- messages[0] = message;
- Mockito.when(snsClient.publish(Mockito.any(PublishRequest.class)))
- .thenReturn(Mockito.any(PublishResult.class));
-
- final int BATCH_SIZE = 50;
- Gson gson = new Gson();
- PublishRequest publishRequest = new PublishRequest();
- for (int i =0; i < messages.length; i+= BATCH_SIZE) {
-
- PubSubInfo[] batch = Arrays.copyOfRange(messages, i, Math.min(messages.length, i + BATCH_SIZE));
- String json = gson.toJson(batch);
-
- // attributes
- Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
- messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
- messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
- headers.addCorrelationIdIfMissing();
- messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getCorrelationId()));
- messageAttributes.put(DpsHeaders.USER_EMAIL, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getUserEmail()));
- messageAttributes.put(DpsHeaders.AUTHORIZATION, new MessageAttributeValue()
- .withDataType("String")
- .withStringValue(headers.getAuthorization()));
- publishRequest.setMessage(json);
- publishRequest.setMessageAttributes(messageAttributes);
- }
-
- // act
- messageBus.publishMessage(headers, message);
-
- // assert
- Mockito.verify(snsClient, Mockito.times(1)).publish(Mockito.eq(publishRequest));
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/S3RecordClientTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/S3RecordClientTest.java
deleted file mode 100644
index e27dc7e1a..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/S3RecordClientTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.storage.Record;
-import org.opengroup.osdu.core.common.model.storage.RecordData;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordProcessing;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.provider.mongodb.util.s3.S3RecordClient;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import javax.inject.Inject;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@RunWith(MockitoJUnitRunner.class)
-@SpringBootTest(classes={StorageApplication.class})
-public class S3RecordClientTest {
-
- @InjectMocks
- @Inject
- @Spy
- private S3RecordClient client;
-
- private String recordsBucketName;
-
- @Mock
- private AmazonS3 s3;
-
- RecordMetadata recordMetadata = new RecordMetadata();
-
- @Before
- public void setUp() {
- initMocks(this);
- recordMetadata.setKind("test-record-id");
- recordMetadata.setId("test-record-id");
- recordMetadata.addGcsPath(1);
- recordMetadata.addGcsPath(2);
- }
-
- @Test
- public void save() {
- // arrange
- RecordProcessing recordProcessing = new RecordProcessing();
- recordProcessing.setRecordMetadata(recordMetadata);
- Record record = new Record();
- record.setId("test-record-id");
- Map<String, Object> data = new HashMap<>();
- data.put("test-data", new Object());
- record.setData(data);
- RecordData recordData = new RecordData(record);
- recordProcessing.setRecordData(recordData);
- String expectedKeyName = recordMetadata.getKind() + "/test-record-id/2";
-
- Mockito.when(s3.putObject(Mockito.eq(recordsBucketName), Mockito.anyString(), Mockito.anyString()))
- .thenReturn(new PutObjectResult());
-
- // act
- client.saveRecord(recordProcessing);
-
- // assert
- Mockito.verify(s3, Mockito.times(1)).putObject(
- Mockito.eq(recordsBucketName), Mockito.eq(expectedKeyName), Mockito.eq("{\"data\":{\"test-data\":{}},\"meta\":null}"));
- }
-
- @Test
- public void getRecordMain(){
- // arrange
- String keyName = "test-key-name";
-
- Mockito.when(s3.getObjectAsString(Mockito.eq(recordsBucketName), Mockito.eq(keyName)))
- .thenReturn("test-result");
-
- // act
- String result = client.getRecord(keyName);
-
- // assert
- Assert.assertEquals("test-result", result);
- }
-
- @Test
- public void getRecordSpecificKeyName(){
- // arrange
- String expectedKeyName = recordMetadata.getKind() + "/" + recordMetadata.getId() + "/1";
-
- Mockito.doReturn("test-result").when(client).getRecord(Mockito.eq(expectedKeyName));
-
- // act
- String result = client.getRecord(recordMetadata, 1L);
-
- // assert
- Mockito.verify(client, Mockito.times(1)).getRecord(
- Mockito.eq(expectedKeyName));
- }
-
- @Test
- public void deleteRecord(){
- // arrange
- String expectedKeyName = recordsBucketName + "/" + recordMetadata.getId();
-
- Mockito.doNothing().when(s3).deleteObject(Mockito.eq(recordsBucketName), Mockito.eq(expectedKeyName));
-
- // act
- client.deleteRecord(recordMetadata);
-
- // assert
- Mockito.verify(s3, Mockito.times(1)).deleteObject(
- Mockito.anyObject());
- }
-
- @Test
- public void checkIfRecordExists(){
- // arrange
- String keyName = recordMetadata.getKind() + "/" + recordMetadata.getId();
-
- Mockito.doReturn(new ListObjectsV2Result()).when(s3)
- .listObjectsV2(Mockito.eq(recordsBucketName), Mockito.eq(keyName));
-
- // act
- boolean result = client.checkIfRecordExists(recordMetadata);
-
- // assert
- Assert.assertTrue(true);
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/UserAccessServiceTest.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/UserAccessServiceTest.java
deleted file mode 100644
index 93511c51c..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/UserAccessServiceTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.mongodb;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.entitlements.EntitlementsException;
-import org.opengroup.osdu.core.common.model.entitlements.GroupInfo;
-import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.storage.StorageApplication;
-import org.opengroup.osdu.storage.provider.mongodb.security.UserAccessService;
-import org.opengroup.osdu.storage.provider.mongodb.util.CacheHelper;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@RunWith(MockitoJUnitRunner.class)
-@SpringBootTest(classes={StorageApplication.class})
-public class UserAccessServiceTest {
-
- @InjectMocks
- private UserAccessService CUT = new UserAccessService();
-
- Acl acl;
-
- RecordMetadata record;
-
- @Before
- public void setUp() {
- initMocks(this);
-
- record = new RecordMetadata();
- record.setUser("not a user");
-
- CacheHelper cacheHelper = Mockito.mock(CacheHelper.class);
- Mockito.when(cacheHelper.getGroupCacheKey(Mockito.anyObject())).thenReturn("test-cache-key");
- Whitebox.setInternalState(CUT, "cacheHelper", cacheHelper);
-
- ICache cache = Mockito.mock(ICache.class);
- Mockito.when(cache.get(Mockito.anyObject())).thenReturn(null);
- Mockito.doNothing().when(cache).put(Mockito.anyObject(), Mockito.anyObject());
- Whitebox.setInternalState(CUT, "cache", cache);
-
- IEntitlementsService entitlementsService = Mockito.mock(IEntitlementsService.class);
- Groups groups = new Groups();
- List<GroupInfo> groupInfos = new ArrayList<>();
- GroupInfo groupInfo = new GroupInfo();
- groupInfo.setName("data.tenant@byoc.local");
- groupInfo.setEmail("data.tenant@byoc.local");
- groupInfos.add(groupInfo);
- groups.setGroups(groupInfos);
-
- try {
- Mockito.when(entitlementsService.getGroups()).thenReturn(groups);
- } catch (EntitlementsException e){
- throw new RuntimeException(e);
- }
-
- IEntitlementsFactory factory = Mockito.mock(IEntitlementsFactory.class);
- Mockito.when(factory.create(Mockito.anyObject())).thenReturn(entitlementsService);
- Whitebox.setInternalState(CUT, "entitlementsFactory", factory);
-
- DpsHeaders dpsHeaders = Mockito.mock(DpsHeaders.class);
- Mockito.when(dpsHeaders.getUserEmail()).thenReturn("notauser@nottheower.com");
- Whitebox.setInternalState(CUT, "dpsHeaders", dpsHeaders);
- }
-
- @Test
- public void userHasAccessToRecord_authorizedUser_ReturnsTrue() {
- // Arrange
- acl = new Acl();
- String[] owners = { "data.tenant@byoc.local" };
- String[] viewers = { "data.tenant@byoc.local" };
- acl.setOwners(owners);
- acl.setViewers(viewers);
-
- record.setAcl(acl);
-
- // Act
- boolean actual = CUT.userHasAccessToRecord(acl);
-
- // Assert
- Assert.assertTrue(actual);
- }
-
- @Test
- public void userHasAccessToRecord_unauthorizedUser_ReturnsFalse() {
- // Arrange
- acl = new Acl();
- acl.setOwners(new String[] {});
- acl.setViewers(new String [] {});
-
- record.setAcl(acl);
-
- // Act
- boolean actual = CUT.userHasAccessToRecord(acl);
-
- // Assert
- Assert.assertFalse(actual);
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/util/Generator.java b/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/util/Generator.java
deleted file mode 100644
index 4b4acd3b8..000000000
--- a/provider/storage-aws-mongodb/src/test/java/org/opengroup/osdu/storage/provider/mongodb/util/Generator.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package org.opengroup.osdu.storage.provider.mongodb.util;
-
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.legal.Legal;
-import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
-import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-import org.opengroup.osdu.core.common.model.storage.RecordState;
-import org.opengroup.osdu.core.common.model.storage.Schema;
-import org.opengroup.osdu.core.common.model.storage.SchemaItem;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-public class Generator {
-
- private static final String[] NAMESET = {"tenant", "source", "foo", "bar", "foobar", "exo", "zoo", "rsq"};
- private static final Random RAND = new Random();
-
- public static String kind() {
- return String.format("opendes:%s:%s:%d.%d.%d",
- NAMESET[RAND.nextInt(NAMESET.length)], NAMESET[RAND.nextInt(NAMESET.length)],
- RAND.nextInt(10), RAND.nextInt(10), RAND.nextInt(10));
- }
-
- public static String path() {
- return path(4);
- }
-
- public static String path(int segments) {
- return IntStream.range(0, segments)
- .mapToObj(i -> (Math.random() > 0.5) ? NAMESET[RAND.nextInt(NAMESET.length)] : String.valueOf(RAND.nextInt(10)))
- .reduce((a, b) -> String.format("%s/%s", a, b))
- .orElse("defPath");
- }
-
- public static Schema schema(String kind) {
- SchemaItem[] schemaItems = new SchemaItem[5];
- for (int i = 0; i < schemaItems.length; i++) {
- SchemaItem item = new SchemaItem();
- item.setKind(Generator.kind());
- item.setPath(Generator.path());
- item.setExt(new HashMap<>());
- item.getExt().put("value", 142 * i);
- schemaItems[i] = item;
- }
- return Schema.builder()
- .schema(schemaItems)
- .kind(kind)
- .ext(new HashMap<>())
- .build();
- }
-
- public static List<RecordMetadata> records(String kind, String user, int size) {
- return IntStream.range(0, size).mapToObj(i -> {
- int id = Math.abs(RAND.nextInt());
- RecordMetadata recordMetadata = new RecordMetadata();
- recordMetadata.setId("opendes:id:" + id);
- recordMetadata.setKind(kind);
-
- Acl recordAcl = new Acl();
- String[] owners = {"data.tenant@byoc.local"};
- String[] viewers = {"data.tenant@byoc.local"};
- recordAcl.setOwners(owners);
- recordAcl.setViewers(viewers);
- recordMetadata.setAcl(recordAcl);
-
- Legal recordLegal = new Legal();
- Set<String> legalTags = new HashSet<>(Collections.singletonList("opendes-storage-" + id));
- recordLegal.setLegaltags(legalTags);
- LegalCompliance status = LegalCompliance.compliant;
- recordLegal.setStatus(status);
- Set<String> otherRelevantDataCountries = new HashSet<>(Collections.singletonList("BR"));
- recordLegal.setOtherRelevantDataCountries(otherRelevantDataCountries);
- recordMetadata.setLegal(recordLegal);
-
- RecordState recordStatus = RecordState.active;
- recordMetadata.setStatus(recordStatus);
-
- recordMetadata.setUser(user);
- return recordMetadata;
- }).collect(Collectors.toList());
- }
-}
diff --git a/provider/storage-aws-mongodb/src/test/resources/application.properties b/provider/storage-aws-mongodb/src/test/resources/application.properties
deleted file mode 100644
index 4b6d8fcc6..000000000
--- a/provider/storage-aws-mongodb/src/test/resources/application.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-LOG_PREFIX=storage
-
-AUTHORIZE_API=notused
-#LEGALTAG_API=${LEGALTAG_BASE_URL}/api/legal/v1
-LEGALTAG_API=legal-tag/api/legal/v1
-
-CRS_API=notused
-
-server.servlet.contextPath=/api/storage/v2/
-logging.level.org.springframework.web=DEBUG
-server.port=8080
-
-## AWS S3 configuration
-aws.region=foo-bar-2000
-aws.s3.endpoint=s3.foo-bar-2000.amazonaws.com
-aws.s3.max-record-threads=2000
-aws.s3.enable-https=true
-S3_DATA_BUCKET=""
-
-aws.mongodb.database.name=test
-
-aws.dynamodb.table.prefix=test
-aws.dynamodb.endpoint=test-endpoint
-aws.sns.s3.bucket-name=foo-bar
-aws.sns.topic.arn=foo-bar
-
-## AWS ElastiCache configuration
-aws.elasticache.cluster.endpoint=localhost
-aws.elasticache.cluster.port=6379
-
-## AWS Lambda configuration
-aws.lambda.get-groups-function-name=test-os-entitlements-GroupsFunction
-
-aws.ssm=false
-aws.parameter.prefix=/osdu/test
\ No newline at end of file
diff --git a/provider/storage-aws/pom.xml b/provider/storage-aws/pom.xml
index 355c46d81..3dabc4b48 100644
--- a/provider/storage-aws/pom.xml
+++ b/provider/storage-aws/pom.xml
@@ -65,12 +65,22 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.14.0-rc2</version>
+ <version>0.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>storage-core</artifactId>
<version>${version.number}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- AWS managed packages -->
<dependency>
@@ -132,8 +142,8 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.10.19</version>
+ <artifactId>mockito-core</artifactId>
+ <version>3.9.0</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -151,6 +161,11 @@
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>de.flapdoodle.embed</groupId>
+ <artifactId>de.flapdoodle.embed.mongo</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
index 8667fb6b0..b4854d86c 100644
--- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
@@ -27,6 +27,7 @@ import org.opengroup.osdu.storage.provider.aws.util.dynamodb.RecordMetadataDoc;
import org.opengroup.osdu.storage.provider.aws.util.dynamodb.SchemaDoc;
import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Repository;
import javax.inject.Inject;
@@ -35,6 +36,8 @@ import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.*;
+@ConditionalOnProperty(prefix = "repository", name = "implementation", havingValue = "dynamodb",
+ matchIfMissing = true)
@Repository
public class QueryRepositoryImpl implements IQueryRepository {
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/RecordsMetadataRepositoryImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/RecordsMetadataRepositoryImpl.java
index ff1e17de3..35e21735f 100644
--- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/RecordsMetadataRepositoryImpl.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/RecordsMetadataRepositoryImpl.java
@@ -27,12 +27,15 @@ import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository
import org.opengroup.osdu.storage.provider.aws.util.dynamodb.LegalTagAssociationDoc;
import org.opengroup.osdu.storage.provider.aws.util.dynamodb.RecordMetadataDoc;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Repository;
import javax.inject.Inject;
import java.io.UnsupportedEncodingException;
import java.util.*;
+@ConditionalOnProperty(prefix = "repository", name = "implementation", havingValue = "dynamodb",
+ matchIfMissing = true)
@Repository
public class RecordsMetadataRepositoryImpl implements IRecordsMetadataRepository<String> {
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/IndexUpdater.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/IndexUpdater.java
new file mode 100644
index 000000000..881b467ab
--- /dev/null
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/IndexUpdater.java
@@ -0,0 +1,61 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.mongo;
+
+import org.opengroup.osdu.core.aws.mongodb.MongoDBMultiClusterFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.index.Index;
+import org.springframework.stereotype.Component;
+
+import java.util.HashSet;
+
+import static org.opengroup.osdu.storage.provider.aws.mongo.MongoDbRecordsMetadataRepository.RECORD_METADATA_PREFIX;
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.DATA_KIND;
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.DATA_LEGAL_LEGALTAGS;
+
+
+@Lazy
+@Component
+public class IndexUpdater {
+
+ private MongoDBMultiClusterFactory mongoDBMultiClusterFactory;
+ private HashSet<String> indexedDataPartitions = new HashSet<>();
+
+ @Autowired
+ public IndexUpdater(MongoDBMultiClusterFactory mongoDBMultiClusterFactory) {
+ this.mongoDBMultiClusterFactory = mongoDBMultiClusterFactory;
+ }
+
+ public void checkIndex(String dataPartitionId) {
+ if (indexedDataPartitions.contains(dataPartitionId)) {
+ return;
+ }
+
+ indexedDataPartitions.add(dataPartitionId);
+ this.updateIndexes(dataPartitionId);
+ }
+
+ private void updateIndexes(String dataPartitionId) {
+ String groupCollection = RECORD_METADATA_PREFIX + dataPartitionId;
+ mongoDBMultiClusterFactory.getHelper(dataPartitionId).ensureIndex(groupCollection, new Index().on(DATA_KIND, Sort.Direction.ASC)
+ .named(DATA_KIND)
+ .on(DATA_LEGAL_LEGALTAGS, Sort.Direction.ASC)
+ .named(DATA_LEGAL_LEGALTAGS)
+ );
+ }
+}
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbQueryRepository.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbQueryRepository.java
new file mode 100644
index 000000000..a439c3e39
--- /dev/null
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbQueryRepository.java
@@ -0,0 +1,99 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.mongo;
+
+import org.opengroup.osdu.core.aws.mongodb.MongoDBMultiClusterFactory;
+import org.opengroup.osdu.core.aws.mongodb.entity.QueryPageResult;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Repository;
+
+import javax.inject.Inject;
+import java.util.stream.Collectors;
+
+import static org.opengroup.osdu.storage.provider.aws.mongo.MongoDbRecordsMetadataRepository.RECORD_METADATA_PREFIX;
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.DATA_KIND;
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.ID;
+
+/**
+ * The type Query repository.
+ */
+@ConditionalOnProperty(prefix = "repository", name = "implementation", havingValue = "mongodb")
+@Repository
+public class MongoDbQueryRepository implements IQueryRepository {
+
+ /**
+ * The Headers.
+ */
+ @Inject
+ DpsHeaders headers;
+
+ @Inject
+ private MongoDBMultiClusterFactory mongoDBMultiClusterFactory;
+
+ @Inject
+ private IndexUpdater indexUpdater;
+
+ private String getDataPartitionId() {
+ return this.headers.getPartitionId();
+ }
+
+
+ /**
+ * Gets all kinds.Not implemented yet
+ *
+ * @param limit the limit
+ * @param cursor the cursor
+ * @return the all kinds
+ */
+ @Override
+ public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
+ return null;
+ }
+
+ /**
+ * Gets all record ids from kind.
+ *
+ * @param kind the kind
+ * @param limit the limit
+ * @param cursor the cursor
+ * @return the all record ids from kind
+ */
+ @Override
+ public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
+ String dataPartitionId = getDataPartitionId();
+ Query query = Query.query(Criteria.where(DATA_KIND).is(kind));
+
+ QueryPageResult<RecordMetadataMongoDBDto> queryPage = mongoDBMultiClusterFactory
+ .getHelper(dataPartitionId)
+ .queryPage(query, ID, cursor, RecordMetadataMongoDBDto.class, limit, getCollection(dataPartitionId));
+
+ DatastoreQueryResult dqr = new DatastoreQueryResult();
+ dqr.setCursor(queryPage.getCursor());
+ dqr.setResults(queryPage.getResults().stream().map(RecordMetadataMongoDBDto::getId).collect(Collectors.toList()));
+ return dqr;
+ }
+
+ private String getCollection(String dataPartitionId) {
+ indexUpdater.checkIndex(dataPartitionId);
+ return RECORD_METADATA_PREFIX + dataPartitionId;
+ }
+}
\ No newline at end of file
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbRecordsMetadataRepository.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbRecordsMetadataRepository.java
new file mode 100644
index 000000000..e3f449b44
--- /dev/null
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/MongoDbRecordsMetadataRepository.java
@@ -0,0 +1,158 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.mongo;
+
+import org.opengroup.osdu.core.aws.mongodb.MongoDBMultiClusterFactory;
+import org.opengroup.osdu.core.aws.mongodb.entity.QueryPageResult;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Repository;
+
+import javax.inject.Inject;
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.DATA_LEGAL_LEGALTAGS;
+import static org.opengroup.osdu.storage.provider.aws.mongo.mongodb.EntityFieldPaths.ID;
+
+/**
+ * The type Records metadata repository.
+ */
+@ConditionalOnProperty(prefix = "repository", name = "implementation", havingValue = "mongodb")
+@Repository
+public class MongoDbRecordsMetadataRepository implements IRecordsMetadataRepository<String> {
+
+ public static final String RECORD_METADATA_PREFIX = "RecordMetadata-";
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Inject
+ private MongoDBMultiClusterFactory mongoDBMultiClusterFactory;
+
+ @Inject
+ private IndexUpdater indexUpdater;
+
+ private String getDataPartitionId() {
+ return this.headers.getPartitionId();
+ }
+
+ /**
+ * Create or update list.
+ *
+ * @param recordsMetadata the records metadata
+ * @return the list
+ */
+ @Override
+ public List<RecordMetadata> createOrUpdate(List<RecordMetadata> recordsMetadata) {
+ String dataPartitionId = getDataPartitionId();
+ String collection = getCollection(dataPartitionId);
+ recordsMetadata.stream()
+ .map(RecordMetadataMongoDBDto::new)
+ .forEach(record -> mongoDBMultiClusterFactory.getHelper(dataPartitionId).save(record, collection));
+ return recordsMetadata;
+ }
+
+ /**
+ * Delete.
+ *
+ * @param id the id
+ */
+ @Override
+ public void delete(String id) {
+ String dataPartitionId = getDataPartitionId();
+ mongoDBMultiClusterFactory.getHelper(dataPartitionId).delete(ID, id, getCollection(dataPartitionId));
+ }
+
+ /**
+ * Get record metadata.
+ *
+ * @param id the id
+ * @return the record metadata
+ */
+ @Override
+ public RecordMetadata get(String id) {
+ String dataPartitionId = getDataPartitionId();
+ RecordMetadataMongoDBDto recordMetadataMongoDBDto = mongoDBMultiClusterFactory
+ .getHelper(dataPartitionId)
+ .getById(id, RecordMetadataMongoDBDto.class, getCollection(dataPartitionId));
+ return recordMetadataMongoDBDto == null ? null : recordMetadataMongoDBDto.getData();
+ }
+
+ /**
+ * Get map.
+ *
+ * @param ids the ids
+ * @return the map
+ */
+ @Override
+ public Map<String, RecordMetadata> get(List<String> ids) {
+ String dataPartitionId = getDataPartitionId();
+ return mongoDBMultiClusterFactory
+ .getHelper(dataPartitionId)
+ .getList(ID, ids, RecordMetadataMongoDBDto.class, getCollection(dataPartitionId))
+ .stream().collect(Collectors.toMap(RecordMetadataMongoDBDto::getId, RecordMetadataMongoDBDto::getData));
+ }
+
+ /**
+ * Query by legal abstract map . simple entry.
+ *
+ * @param legalTagName the legal tag name
+ * @param status the status
+ * @param limit the limit
+ * @return the abstract map . simple entry
+ */
+ @Override
+ public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegal(String legalTagName, LegalCompliance status, int limit) {
+ return null;
+ }
+
+ /**
+ * Query by legal tag name abstract map . simple entry.
+ *
+ * @param legalTagName the legal tag name
+ * @param limit the limit
+ * @param cursor the cursor
+ * @return the abstract map . simple entry
+ */
+ @Override
+ public AbstractMap.SimpleEntry<String, List<RecordMetadata>> queryByLegalTagName(
+ String legalTagName, int limit, String cursor) {
+ String dataPartitionId = getDataPartitionId();
+ Query query = Query.query(Criteria.where(DATA_LEGAL_LEGALTAGS).is(legalTagName));
+ QueryPageResult<RecordMetadataMongoDBDto> queryPage = mongoDBMultiClusterFactory
+ .getHelper(dataPartitionId)
+ .queryPage(query, ID, cursor, RecordMetadataMongoDBDto.class, limit, getCollection(dataPartitionId));
+
+ return new AbstractMap.SimpleEntry<>(queryPage.getCursor(),
+ queryPage.getResults().stream()
+ .map(RecordMetadataMongoDBDto::getData)
+ .collect(Collectors.toList()));
+ }
+
+ private String getCollection(String dataPartitionId) {
+ indexUpdater.checkIndex(dataPartitionId);
+ return RECORD_METADATA_PREFIX + dataPartitionId;
+ }
+}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/ServiceAccountJwtClientImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/dto/RecordMetadataMongoDBDto.java
similarity index 53%
rename from provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/ServiceAccountJwtClientImpl.java
rename to provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/dto/RecordMetadataMongoDBDto.java
index 6bb0351e8..42f271db7 100644
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/ServiceAccountJwtClientImpl.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/dto/RecordMetadataMongoDBDto.java
@@ -1,4 +1,5 @@
-// Copyright © 2020 Amazon Web Services
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,15 +13,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.storage.provider.mongodb.util;
+package org.opengroup.osdu.storage.provider.aws.mongo.dto;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.springframework.stereotype.Component;
+import org.opengroup.osdu.core.aws.mongodb.entity.BasicMongoDBDoc;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
-@Component
-public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
- @Override
- public String getIdToken(String tenantName){
- return "TBD-should-not-pass-around";
+public class RecordMetadataMongoDBDto extends BasicMongoDBDoc<RecordMetadata> {
+ public RecordMetadataMongoDBDto(RecordMetadata data) {
+ super(data, data.getId());
}
}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SomeBasicInterfaceImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/EntityFieldPaths.java
similarity index 57%
rename from provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SomeBasicInterfaceImpl.java
rename to provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/EntityFieldPaths.java
index 16036c480..81d50291e 100644
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/SomeBasicInterfaceImpl.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/EntityFieldPaths.java
@@ -1,4 +1,5 @@
-// Copyright © 2020 Amazon Web Services
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,15 +13,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.storage.provider.mongodb;
+package org.opengroup.osdu.storage.provider.aws.mongo.mongodb;
-import org.opengroup.osdu.storage.provider.interfaces.ISomeBasicInterface;
-import org.springframework.stereotype.Component;
-
-@Component
-public class SomeBasicInterfaceImpl implements ISomeBasicInterface {
- @Override
- public String hello() {
- return "Hello from MongoDB!";
- }
+public class EntityFieldPaths {
+ public static final String ID = "_id";
+ public static final String DATA_KIND = "data.kind";
+ public static final String DATA_LEGAL_LEGALTAGS = "data.legal.legaltags";
}
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
new file mode 100644
index 000000000..33e79b626
--- /dev/null
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
@@ -0,0 +1,27 @@
+package org.opengroup.osdu.storage.provider.aws.mongo.mongodb;
+
+import org.opengroup.osdu.core.aws.mongodb.MultiClusteredConfigReader;
+import org.opengroup.osdu.core.aws.mongodb.config.MongoProperties;
+import org.opengroup.osdu.core.aws.mongodb.config.MongoPropertiesReader;
+import org.opengroup.osdu.core.aws.partition.PartitionInfoAws;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+@Lazy
+@Component
+public class MultiClusteredConfigReaderStorage implements MultiClusteredConfigReader {
+
+ private final MongoPropertiesReader propertiesReader;
+
+ @Autowired
+ public MultiClusteredConfigReaderStorage(MongoPropertiesReader propertiesReader) {
+ this.propertiesReader = propertiesReader;
+ }
+
+ @Override
+ public MongoProperties readProperties(PartitionInfoAws partitionInfoAws) {
+ return propertiesReader.getProperties();
+ }
+
+}
diff --git a/provider/storage-aws/src/main/resources/application.properties b/provider/storage-aws/src/main/resources/application.properties
index 0015672d2..a7c78e74a 100644
--- a/provider/storage-aws/src/main/resources/application.properties
+++ b/provider/storage-aws/src/main/resources/application.properties
@@ -68,3 +68,24 @@ aws.parameter.prefix=/osdu/${ENVIRONMENT}
aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT:null}
aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT:null}
aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY:null}
+
+# values mongo/dinamo for choose any condition from implementation
+repository.implementation=${STORAGE_SERVICE_REPOSITORY_IMPLEMENTATION:dynamodb}
+
+#MongoDB config
+osdu.mongodb.database=${ENVIRONMENT}_osdu_storage
+osdu.mongodb.authDatabase=${MONGODB_AUTH_DATABASE:admin}
+osdu.mongodb.endpoint=${MONGODB_ENDPOINT:}
+osdu.mongodb.port=${MONGODB_PORT:27017}
+osdu.mongodb.username=${MONGODB_USERNAME:empty}
+osdu.mongodb.password=${MONGODB_PASSWORD:empty}
+osdu.mongodb.retryWrites=${MONGODB_RETRY_WRITES:true}
+osdu.mongodb.writeMode=${MONGODB_WRITE_MODE:majority}
+osdu.mongodb.useSrvEndpoint=${MONGODB_USE_SRV_ENDPOINT:true}
+osdu.mongodb.enableTLS=${MONGODB_ENABLE_TLS:false}
+osdu.mongodb.maxPoolSize=${MONGODB_MAX_POOL_SIZE:300}
+osdu.mongodb.readPreference=${MONGODB_READ_PREFERENCE:nearest}
+osdu.mongodb.maxIdleTimeMS=${MONGODB_MAX_IDLE_TIME_MS:30000}
+
+#Tomcat limits
+server.tomcat.threads.max=${TOMCAT_THREADS_MAX:300}
\ No newline at end of file
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/CloudStorageTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/CloudStorageTest.java
index b007dccc9..3904ebad6 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/CloudStorageTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/CloudStorageTest.java
@@ -15,7 +15,7 @@
package org.opengroup.osdu.storage.provider.aws.api;
import com.google.gson.Gson;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.aws.s3.S3ClientFactory;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
@@ -45,7 +45,7 @@ import java.util.*;
import java.util.concurrent.*;
import static org.apache.commons.codec.binary.Base64.encodeBase64;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes={StorageApplication.class})
@@ -77,11 +77,11 @@ public class CloudStorageTest {
@Inject
private JaxRsDpsLog logger;
- @Mock
+ @Mock
private DpsHeaders headers;
private String dataPartition = "dummyPartitionName";
-
+
String userId = "test-user-id";
RecordMetadata record = new RecordMetadata();
@@ -89,12 +89,12 @@ public class CloudStorageTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
record.setId("test-record-id");
record.addGcsPath(1);
records.add(record);
-
- Mockito.when(headers.getPartitionIdWithFallbackToAccountId()).thenReturn(dataPartition);
+
+ Mockito.when(headers.getPartitionIdWithFallbackToAccountId()).thenReturn(dataPartition);
}
@@ -164,12 +164,6 @@ public class CloudStorageTest {
Map<String, String> expectedResp = new HashMap<>();
expectedResp.put("test-record-id", "{data:test-data}");
- RecordMetadata recordMetadata = new RecordMetadata();
-
- Mockito.when(recordsMetadataRepository.get("test-record-id")).thenReturn(recordMetadata);
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.anyObject()))
- .thenReturn(true);
-
Mockito.when(recordsUtil.getRecordsValuesById(Mockito.eq(map)))
.thenReturn(expectedResp);
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java
index 88282c012..bb9e59f3e 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/LegalComplianceChangeServiceAWSImplTest.java
@@ -21,7 +21,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.legal.Legal;
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/MessageBusImplTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/MessageBusImplTest.java
index 18a7b0f96..a5a4f6e0d 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/MessageBusImplTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/MessageBusImplTest.java
@@ -33,13 +33,14 @@ import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
import org.opengroup.osdu.storage.provider.aws.MessageBusImpl;
import org.springframework.boot.test.context.SpringBootTest;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
+
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes = {StorageApplication.class})
@@ -56,7 +57,7 @@ public class MessageBusImplTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
}
@Test
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
index 13948d5ea..37cf239ba 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
@@ -21,7 +21,7 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
@@ -41,7 +41,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes = {StorageApplication.class})
@@ -63,12 +63,9 @@ public class QueryRepositoryTest {
@Before
public void setUp() {
- initMocks(this);
-
+ openMocks(this);
Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(DpsHeaders.class), Mockito.any()))
.thenReturn(queryHelper);
- Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(String.class), Mockito.any()))
- .thenReturn(queryHelper);
}
@Test
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/RecordsMetadataRepositoryTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/RecordsMetadataRepositoryTest.java
index 7d9ac3352..8d9201018 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/RecordsMetadataRepositoryTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/RecordsMetadataRepositoryTest.java
@@ -14,7 +14,7 @@
package org.opengroup.osdu.storage.provider.aws.api;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
import org.opengroup.osdu.core.common.model.legal.Legal;
import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
@@ -40,7 +40,7 @@ import org.springframework.boot.test.context.SpringBootTest;
import java.util.*;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes={StorageApplication.class})
@@ -65,12 +65,9 @@ public class RecordsMetadataRepositoryTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(DpsHeaders.class), Mockito.any()))
.thenReturn(queryHelper);
- Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(String.class), Mockito.any()))
- .thenReturn(queryHelper);
-
}
@Test
@@ -116,8 +113,6 @@ public class RecordsMetadataRepositoryTest {
Mockito.doNothing().when(queryHelper).save(Mockito.eq(expectedRmd));
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.eq(recordAcl))).thenReturn(true);
-
// Act
repo.createOrUpdate(recordsMetadata);
@@ -175,8 +170,6 @@ public class RecordsMetadataRepositoryTest {
Mockito.when(queryHelper.loadByPrimaryKey(Mockito.eq(RecordMetadataDoc.class), Mockito.anyString()))
.thenReturn(expectedRmd);
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.eq(recordAcl))).thenReturn(true);
-
// Act
RecordMetadata recordMetadata = repo.get(id);
@@ -239,8 +232,6 @@ public class RecordsMetadataRepositoryTest {
groupInfos.add(groupInfo);
groups.setGroups(groupInfos);
- Mockito.when(userAccessService.userHasAccessToRecord(Mockito.eq(recordAcl))).thenReturn(true);
-
// Act
Map<String, RecordMetadata> recordsMetadata = repo.get(ids);
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/S3RecordClientTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/S3RecordClientTest.java
index 7cefd1a45..420e0c042 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/S3RecordClientTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/S3RecordClientTest.java
@@ -17,7 +17,7 @@ package org.opengroup.osdu.storage.provider.aws.api;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.PutObjectResult;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.aws.s3.S3ClientFactory;
@@ -45,7 +45,7 @@ import javax.inject.Inject;
import java.util.*;
import static org.apache.commons.codec.binary.Base64.encodeBase64;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes={StorageApplication.class})
@@ -82,28 +82,18 @@ public class S3RecordClientTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
recordMetadata.setKind("test-record-id");
recordMetadata.setId("test-record-id");
recordMetadata.addGcsPath(1);
recordMetadata.addGcsPath(2);
- Mockito.when(headers.getPartitionIdWithFallbackToAccountId()).thenReturn(dataPartition);
-
- Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(DpsHeaders.class), Mockito.any()))
- .thenReturn(queryHelper);
- Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(String.class), Mockito.any()))
- .thenReturn(queryHelper);
-
Mockito.when(s3ClientWithBucket.getS3Client()).thenReturn(s3);
Mockito.when(s3ClientWithBucket.getBucketName()).thenReturn(recordsBucketName);
- Mockito.when(s3ClientFactory.getS3ClientForPartition(Mockito.anyString(), Mockito.anyString()))
- .thenReturn(s3ClientWithBucket);
-
- Mockito.when(s3ClientFactory.getS3ClientForPartition(Mockito.any(DpsHeaders.class), Mockito.anyString()))
+ Mockito.when(s3ClientFactory.getS3ClientForPartition(Mockito.nullable(String.class), Mockito.nullable(String.class)))
.thenReturn(s3ClientWithBucket);
-
+
}
@Test
@@ -166,8 +156,6 @@ public class S3RecordClientTest {
// arrange
String expectedKeyName = recordsBucketName + "/" + recordMetadata.getId();
- Mockito.doNothing().when(s3).deleteObject(Mockito.eq(recordsBucketName), Mockito.eq(expectedKeyName));
-
// act
client.deleteRecord(recordMetadata, dataPartition);
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/SchemaRepositoryTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/SchemaRepositoryTest.java
index 12b3944e3..be8cffb60 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/SchemaRepositoryTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/SchemaRepositoryTest.java
@@ -31,12 +31,12 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.boot.test.context.SpringBootTest;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.storage.provider.aws.SchemaRepositoryImpl;
import java.util.*;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes={StorageApplication.class})
@@ -58,12 +58,10 @@ public class SchemaRepositoryTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(DpsHeaders.class), Mockito.any()))
.thenReturn(queryHelper);
- Mockito.when(queryHelperFactory.getQueryHelperForPartition(Mockito.any(String.class), Mockito.any()))
- .thenReturn(queryHelper);
}
@Test
@@ -86,8 +84,6 @@ public class SchemaRepositoryTest {
expectedSd.setSchemaItems(Arrays.asList(schema.getSchema()));
expectedSd.setDataPartitionId(dataPartitionId);
- Mockito.when(queryHelper.loadByPrimaryKey(Mockito.eq(SchemaDoc.class), Mockito.anyString()))
- .thenReturn(null);
Mockito.when(headers.getPartitionId()).thenReturn(dataPartitionId);
Mockito.doNothing().when(queryHelper).save(Mockito.eq(expectedSd));
@@ -133,7 +129,6 @@ public class SchemaRepositoryTest {
SchemaDoc expectedSd = new SchemaDoc();
expectedSd.setKind(kind);
- Mockito.doNothing().when(queryHelper).deleteByObject(expectedSd);
Mockito.doNothing().when(queryHelper).deleteByPrimaryKey(Mockito.eq(SchemaDoc.class), Mockito.anyString());
// Act
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/UserAccessServiceTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/UserAccessServiceTest.java
index 5c978c04b..95efbb1cc 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/UserAccessServiceTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/UserAccessServiceTest.java
@@ -21,8 +21,8 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.Whitebox;
-import org.mockito.runners.MockitoJUnitRunner;
+import org.powermock.reflect.Whitebox;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
@@ -44,7 +44,7 @@ import org.springframework.boot.test.context.SpringBootTest;
import java.util.ArrayList;
import java.util.List;
-import static org.mockito.MockitoAnnotations.initMocks;
+import static org.mockito.MockitoAnnotations.openMocks;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest(classes={StorageApplication.class})
@@ -62,18 +62,15 @@ public class UserAccessServiceTest {
@Before
public void setUp() {
- initMocks(this);
+ openMocks(this);
record = new RecordMetadata();
record.setUser("not a user");
CacheHelper cacheHelper = Mockito.mock(CacheHelper.class);
- Mockito.when(cacheHelper.getGroupCacheKey(Mockito.anyObject())).thenReturn("test-cache-key");
Whitebox.setInternalState(CUT, "cacheHelper", cacheHelper);
GroupCache cache = Mockito.mock(GroupCache.class);
- Mockito.when(cache.get(Mockito.anyObject())).thenReturn(null);
- Mockito.doNothing().when(cache).put(Mockito.anyObject(), Mockito.anyObject());
Whitebox.setInternalState(CUT, "cache", cache);
IEntitlementsService entitlementsService = Mockito.mock(IEntitlementsService.class);
@@ -85,20 +82,13 @@ public class UserAccessServiceTest {
groupInfos.add(groupInfo);
groups.setGroups(groupInfos);
- try {
- Mockito.when(entitlementsService.getGroups()).thenReturn(groups);
- } catch (EntitlementsException e){
- throw new RuntimeException(e);
- }
Mockito.when(entitlementsExtension.getGroups(Mockito.any())).thenReturn(groups);
IEntitlementsFactory factory = Mockito.mock(IEntitlementsFactory.class);
- Mockito.when(factory.create(Mockito.anyObject())).thenReturn(entitlementsService);
Whitebox.setInternalState(CUT, "entitlementsFactory", factory);
DpsHeaders dpsHeaders = Mockito.mock(DpsHeaders.class);
- Mockito.when(dpsHeaders.getUserEmail()).thenReturn("notauser@nottheower.com");
Whitebox.setInternalState(CUT, "dpsHeaders", dpsHeaders);
}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/QueryRepositoryMongoTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/QueryRepositoryMongoTest.java
new file mode 100644
index 000000000..6360f7c8f
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/QueryRepositoryMongoTest.java
@@ -0,0 +1,91 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo;
+
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.configuration.StorageTestConfig;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.DbUtil;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.ParentUtil;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.RecordMetadataGenerator;
+import org.opengroup.osdu.storage.provider.aws.mongo.MongoDbQueryRepository;
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest;
+import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+@DataMongoTest
+@RunWith(SpringRunner.class)
+@SpringJUnitConfig(classes = StorageTestConfig.class)
+public class QueryRepositoryMongoTest extends ParentUtil {
+
+ @Autowired
+ private MongoDbQueryRepository queryRepository;
+
+ @Test
+ public void getAllRecordIdsFromKind() {
+ //given
+ String someKind = "someKind";
+ List<RecordMetadataMongoDBDto> firstList = RecordMetadataGenerator.generate(27, someKind);
+ String otherKind = "otherKind";
+ List<RecordMetadataMongoDBDto> secondList = RecordMetadataGenerator.generate(15, otherKind);
+ mongoTemplateHelper.insert(firstList, DbUtil.DATA_PARTITION);
+ mongoTemplateHelper.insert(secondList, DbUtil.DATA_PARTITION);
+ List<RecordMetadataMongoDBDto> all = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(firstList.size() + secondList.size(), all.size());
+
+ //when
+ DatastoreQueryResult firstPageRecords = queryRepository.getAllRecordIdsFromKind(someKind, 5, null);
+ String cursor1 = firstPageRecords.getCursor();
+ List<String> firstPageRecordsIds = firstPageRecords.getResults();
+
+ DatastoreQueryResult secondPageRecords = queryRepository.getAllRecordIdsFromKind(someKind, 20, cursor1);
+ String cursor2 = secondPageRecords.getCursor();
+ List<String> secondPageRecordsIds = secondPageRecords.getResults();
+
+ DatastoreQueryResult lastPageRecords = queryRepository.getAllRecordIdsFromKind(someKind, 5, cursor2);
+ String cursor3 = lastPageRecords.getCursor();
+ List<String> lastPageRecordsIds = lastPageRecords.getResults();
+
+
+ DatastoreQueryResult otherRecords = queryRepository.getAllRecordIdsFromKind(otherKind, 50, null);
+ String cursor4 = otherRecords.getCursor();
+ List<String> otherResults = otherRecords.getResults();
+
+
+ //then
+ assertNotNull(cursor1);
+ assertEquals(5, firstPageRecordsIds.size());
+
+ assertNotNull(cursor2);
+ assertEquals(20, secondPageRecordsIds.size());
+
+ assertNull(cursor3);
+ assertEquals(2, lastPageRecordsIds.size());
+
+ assertNull(cursor4);
+ assertEquals(15, otherResults.size());
+ }
+}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/RecordsMetadataRepositoryMongoTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/RecordsMetadataRepositoryMongoTest.java
new file mode 100644
index 000000000..e62aeb30f
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/RecordsMetadataRepositoryMongoTest.java
@@ -0,0 +1,230 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo;
+
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.configuration.StorageTestConfig;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.DbUtil;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.ParentUtil;
+import org.opengroup.osdu.storage.provider.aws.api.mongo.util.RecordMetadataGenerator;
+import org.opengroup.osdu.storage.provider.aws.mongo.MongoDbRecordsMetadataRepository;
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest;
+import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@DataMongoTest
+@RunWith(SpringRunner.class)
+@SpringJUnitConfig(classes = StorageTestConfig.class)
+public class RecordsMetadataRepositoryMongoTest extends ParentUtil {
+
+ @Autowired
+ private MongoDbRecordsMetadataRepository recordsMetadataRepository;
+
+ @Test
+ public void createRecordMetadata() {
+ //given
+ List<RecordMetadataMongoDBDto> allFromHelper = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(0, allFromHelper.size());
+
+ List<RecordMetadata> recordMetadataList = RecordMetadataGenerator.generate(5, "legalTag").stream().map(RecordMetadataMongoDBDto::getData).collect(Collectors.toList());
+
+ //when
+ List<RecordMetadata> recordMetadataListFromRepo = recordsMetadataRepository.createOrUpdate(recordMetadataList);
+
+ //then
+ allFromHelper = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(recordMetadataList.size(), allFromHelper.size());
+ List<String> idsFromRepo = recordMetadataListFromRepo.stream().map(RecordMetadata::getId).collect(Collectors.toList());
+ List<String> idsFromDb = allFromHelper.stream().map(RecordMetadataMongoDBDto::getId).collect(Collectors.toList());
+ assertTrue(idsFromDb.containsAll(idsFromRepo));
+ }
+
+ @Test
+ public void createAndUpdateRecordMetadata() {
+ //given
+ List<RecordMetadataMongoDBDto> existsList = RecordMetadataGenerator.generate(5, "legalTag");
+ mongoTemplateHelper.insert(existsList, DbUtil.DATA_PARTITION);
+ List<RecordMetadataMongoDBDto> allFromHelper = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(existsList.size(), allFromHelper.size());
+ assertEquals(RecordState.active, allFromHelper.stream().findAny().get().getData().getStatus());
+
+ List<RecordMetadata> existsListAfterChange = existsList.stream().map(recordMetadataMongoDto -> {
+ RecordMetadata data = recordMetadataMongoDto.getData();
+ data.setStatus(RecordState.deleted);
+ return data;
+ }).collect(Collectors.toList());
+
+
+ int notExistrecordsCount = 8;
+ List<RecordMetadata> notExistsList = RecordMetadataGenerator.generate(notExistrecordsCount, "legalTag").stream().map(RecordMetadataMongoDBDto::getData).collect(Collectors.toList());
+
+ notExistsList.addAll(existsListAfterChange);
+ assertEquals(existsListAfterChange.size() + notExistrecordsCount, notExistsList.size());
+
+ //when
+ List<RecordMetadata> recordMetadataListFromRepo = recordsMetadataRepository.createOrUpdate(notExistsList);
+
+ //then
+ allFromHelper = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(notExistsList.size(), allFromHelper.size());
+ List<String> idsFromRepo = recordMetadataListFromRepo.stream().map(RecordMetadata::getId).collect(Collectors.toList());
+ List<String> idsFromDb = allFromHelper.stream().map(RecordMetadataMongoDBDto::getId).collect(Collectors.toList());
+ assertTrue(idsFromDb.containsAll(idsFromRepo));
+ assertEquals(RecordState.deleted, allFromHelper.stream().filter(recordMetadataMongoDto -> existsListAfterChange.stream().map(RecordMetadata::getId).collect(Collectors.toList()).contains(recordMetadataMongoDto.getId())).findAny().get().getData().getStatus());
+ }
+
+ @Test
+ public void delete() {
+ // given
+ RecordMetadataMongoDBDto recordMetadataMongoDBDto = RecordMetadataGenerator.create();
+ String id = recordMetadataMongoDBDto.getData().getId();
+
+ assertNotNull(mongoTemplateHelper.insert(recordMetadataMongoDBDto));
+ List<RecordMetadataMongoDBDto> recordMetadataMongoDBDtos = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(1, recordMetadataMongoDBDtos.size());
+
+ //when
+ recordsMetadataRepository.delete(id);
+
+ //then
+ recordMetadataMongoDBDtos = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(0, recordMetadataMongoDBDtos.size());
+ }
+
+ @Test
+ public void deleteIfNotExists() {
+ // given
+ List<RecordMetadataMongoDBDto> recordMetadataMongoDBDtos = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(0, recordMetadataMongoDBDtos.size());
+
+ //when
+ recordsMetadataRepository.delete("anyId");
+
+ //then
+ recordMetadataMongoDBDtos = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(0, recordMetadataMongoDBDtos.size());
+ }
+
+ @Test
+ public void get() {
+ // given
+ RecordMetadataMongoDBDto recordMetadataMongoDBDto = RecordMetadataGenerator.create();
+ String id = recordMetadataMongoDBDto.getData().getId();
+
+ assertNotNull(mongoTemplateHelper.insert(recordMetadataMongoDBDto));
+
+ //when
+ RecordMetadata recordMetadataFromRepo = recordsMetadataRepository.get(id);
+
+ //then
+ assertEquals(id, recordMetadataFromRepo.getId());
+ }
+
+ @Test
+ public void getNotFound() {
+ // given
+ List<RecordMetadataMongoDBDto> all = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(0, all.size());
+
+ //when
+ RecordMetadata recordMetadataFromRepo = recordsMetadataRepository.get(RECORD_ID);
+
+ //then
+ assertNull(recordMetadataFromRepo);
+ }
+
+ @Test
+ public void getList() {
+ //given
+ List<RecordMetadataMongoDBDto> firstList = RecordMetadataGenerator.generate(5, "legalTag");
+ List<RecordMetadataMongoDBDto> secondList = RecordMetadataGenerator.generate(5, "otherLegalTag");
+ mongoTemplateHelper.insert(firstList, DbUtil.DATA_PARTITION);
+ mongoTemplateHelper.insert(secondList, DbUtil.DATA_PARTITION);
+ List<RecordMetadataMongoDBDto> all = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(firstList.size() + secondList.size(), all.size());
+
+ //when
+ Map<String, RecordMetadata> stringRecordMetadataMap = recordsMetadataRepository.get(firstList.stream().map(RecordMetadataMongoDBDto::getId).collect(Collectors.toList()));
+
+ //then
+ stringRecordMetadataMap.forEach((key, value) -> {
+ assertEquals(key, value.getId());
+ assertTrue(firstList.stream().map(RecordMetadataMongoDBDto::getId).anyMatch(id -> id.equals(key)));
+ assertTrue(secondList.stream().map(RecordMetadataMongoDBDto::getId).noneMatch(id -> id.equals(key)));
+ });
+ }
+
+ @Test
+ public void queryByLegalTagName() {
+ //given
+ String legalTagName = "legalTag";
+ List<RecordMetadataMongoDBDto> firstList = RecordMetadataGenerator.generate(27, legalTagName);
+ String otherLegalTag = "otherLegalTag";
+ List<RecordMetadataMongoDBDto> secondList = RecordMetadataGenerator.generate(15, otherLegalTag);
+ mongoTemplateHelper.insert(firstList, DbUtil.DATA_PARTITION);
+ mongoTemplateHelper.insert(secondList, DbUtil.DATA_PARTITION);
+ List<RecordMetadataMongoDBDto> all = mongoTemplateHelper.findAll(DbUtil.DATA_PARTITION);
+ assertEquals(firstList.size() + secondList.size(), all.size());
+
+ //when
+ legalTagName = legalTagName + "0";
+ AbstractMap.SimpleEntry<String, List<RecordMetadata>> firstPage = recordsMetadataRepository.queryByLegalTagName(legalTagName, 5, null);
+ String cursor1 = firstPage.getKey();
+
+ List<RecordMetadata> firstPageRecords = firstPage.getValue();
+ AbstractMap.SimpleEntry<String, List<RecordMetadata>> secondPage = recordsMetadataRepository.queryByLegalTagName(legalTagName, 20, cursor1);
+ String cursor2 = secondPage.getKey();
+
+ List<RecordMetadata> secondPageRecords = secondPage.getValue();
+ AbstractMap.SimpleEntry<String, List<RecordMetadata>> lastPage = recordsMetadataRepository.queryByLegalTagName(legalTagName, 5, cursor2);
+ String cursor3 = lastPage.getKey();
+ List<RecordMetadata> lastPageRecords = lastPage.getValue();
+
+ otherLegalTag = otherLegalTag + "1";
+ AbstractMap.SimpleEntry<String, List<RecordMetadata>> otherLegalTags = recordsMetadataRepository.queryByLegalTagName(otherLegalTag, 50, null);
+ String cursor4 = otherLegalTags.getKey();
+ List<RecordMetadata> otherLegalTagsValue = otherLegalTags.getValue();
+
+ //then
+ assertNotNull(cursor1);
+ assertEquals(5, firstPageRecords.size());
+
+ assertNotNull(cursor2);
+ assertEquals(20, secondPageRecords.size());
+
+ assertNull(cursor3);
+ assertEquals(2, lastPageRecords.size());
+
+ assertNull(cursor4);
+ assertEquals(15, otherLegalTagsValue.size());
+ }
+}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/configuration/StorageTestConfig.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/configuration/StorageTestConfig.java
new file mode 100644
index 000000000..fe7654698
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/configuration/StorageTestConfig.java
@@ -0,0 +1,91 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo.configuration;
+
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.aws.mongodb.MongoDBSimpleFactory;
+import org.opengroup.osdu.core.aws.mongodb.MultiClusteredConfigReader;
+import org.opengroup.osdu.core.aws.mongodb.config.MongoProperties;
+import org.opengroup.osdu.core.common.storage.IngestionService;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.storage.conversion.CrsConversionService;
+import org.opengroup.osdu.storage.provider.aws.CloudStorageImpl;
+import org.opengroup.osdu.storage.provider.aws.MessageBusImpl;
+import org.opengroup.osdu.storage.provider.aws.SchemaRepositoryImpl;
+import org.opengroup.osdu.storage.provider.aws.SomeBasicInterfaceImpl;
+import org.opengroup.osdu.storage.provider.aws.cache.GroupCache;
+import org.opengroup.osdu.storage.provider.aws.cache.LegalTagCache;
+import org.opengroup.osdu.storage.provider.aws.mongo.mongodb.MultiClusteredConfigReaderStorage;
+import org.opengroup.osdu.storage.provider.aws.security.UserAccessService;
+import org.opengroup.osdu.storage.provider.aws.service.BatchServiceAWSImpl;
+import org.opengroup.osdu.storage.service.BulkUpdateRecordServiceImpl;
+import org.opengroup.osdu.storage.service.DataAuthorizationService;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import static org.mockito.ArgumentMatchers.any;
+
+
+@TestConfiguration
+public class StorageTestConfig {
+ private final MongoProperties properties = MongoProperties.builder().
+ endpoint("localhost:27019").
+ databaseName("test")
+ .build();
+
+ @Bean
+ @Primary
+ public MultiClusteredConfigReader configReaderStorage() {
+ MultiClusteredConfigReaderStorage multiClusteredConfigReaderStorage = Mockito.mock(MultiClusteredConfigReaderStorage.class);
+ Mockito.doReturn(properties).when(multiClusteredConfigReaderStorage).readProperties(any());
+ return multiClusteredConfigReaderStorage;
+ }
+
+ @MockBean
+ public CloudStorageImpl cloudStorage;
+ @MockBean
+ public MessageBusImpl messageBus;
+ @MockBean
+ public SchemaRepositoryImpl schemaRepository;
+ @MockBean
+ public SomeBasicInterfaceImpl someBasicInterface;
+ @MockBean
+ public GroupCache groupCache;
+ @MockBean
+ public LegalTagCache legalTagCache;
+ @MockBean
+ public UserAccessService userAccessService;
+ @MockBean
+ public BatchServiceAWSImpl batchServiceAWS;
+ @MockBean
+ public CrsConversionService crsConversionService;
+ @MockBean
+ public BulkUpdateRecordServiceImpl bulkUpdateRecordService;
+ @MockBean
+ public DataAuthorizationService dataAuthorizationService;
+ @MockBean
+ public IngestionService ingestionService;
+ @MockBean
+ public IServiceAccountJwtClient serviceAccountJwtClient;
+
+ @Bean
+ public MongoTemplate createMongoTemplate(MongoDBSimpleFactory dbSimpleFactory) {
+ return dbSimpleFactory.mongoTemplate(properties);
+ }
+}
diff --git a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/CallableResult.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/DbUtil.java
similarity index 60%
rename from provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/CallableResult.java
rename to provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/DbUtil.java
index 040a3b9c9..2e4b15e07 100644
--- a/provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/util/s3/CallableResult.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/DbUtil.java
@@ -1,4 +1,5 @@
-// Copyright © 2020 Amazon Web Services
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,9 +13,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.storage.provider.mongodb.util.s3;
+package org.opengroup.osdu.storage.provider.aws.api.mongo.util;
+
+public abstract class DbUtil {
+
+ public static final String DATA_PARTITION = "osdu_storage_data_partition";
+ public static final String RECORD_ID = "recordId";
-public enum CallableResult {
- Pass,
- Fail
}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/MongoTemplateHelper.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/MongoTemplateHelper.java
new file mode 100644
index 000000000..debef6d04
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/MongoTemplateHelper.java
@@ -0,0 +1,53 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo.util;
+
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import java.util.Collection;
+import java.util.List;
+
+import static org.opengroup.osdu.storage.provider.aws.mongo.MongoDbRecordsMetadataRepository.RECORD_METADATA_PREFIX;
+
+
+public final class MongoTemplateHelper extends DbUtil {
+ private final MongoTemplate mongoTemplate;
+
+ public MongoTemplateHelper(MongoTemplate mongoTemplate) {
+ this.mongoTemplate = mongoTemplate;
+ }
+
+ public RecordMetadataMongoDBDto insert(RecordMetadataMongoDBDto recordMetadata) {
+ return this.insert(recordMetadata, DbUtil.DATA_PARTITION);
+ }
+
+ public void insert(Collection<? extends RecordMetadataMongoDBDto> recordMetadataMongoDtos, String collection) {
+ recordMetadataMongoDtos.forEach(t -> this.insert(t, collection));
+ }
+
+ public RecordMetadataMongoDBDto insert(RecordMetadataMongoDBDto recordMetadata, String dataPartition) {
+ return this.mongoTemplate.insert(recordMetadata, RECORD_METADATA_PREFIX + dataPartition);
+ }
+
+ public List<RecordMetadataMongoDBDto> findAll(String dataPartition) {
+ return this.mongoTemplate.findAll(RecordMetadataMongoDBDto.class, RECORD_METADATA_PREFIX + dataPartition);
+ }
+
+ public void dropCollections() {
+ this.mongoTemplate.getCollectionNames().forEach(this.mongoTemplate::dropCollection);
+ }
+}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/ParentUtil.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/ParentUtil.java
new file mode 100644
index 000000000..960f4a082
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/ParentUtil.java
@@ -0,0 +1,64 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo.util;
+
+import org.junit.Rule;
+import org.junit.rules.ExternalResource;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.aws.partition.PartitionInfoAws;
+import org.opengroup.osdu.core.aws.partition.PartitionServiceClientWithCache;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.partition.Property;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import static org.mockito.ArgumentMatchers.anyString;
+
+public abstract class ParentUtil extends DbUtil {
+
+ public MongoTemplateHelper mongoTemplateHelper;
+
+ @MockBean
+ private DpsHeaders headers;
+ @MockBean
+ private PartitionServiceClientWithCache partitionServiceClient;
+
+ @Rule
+ public ExternalResource resource = new ExternalResource() {
+ @Override
+ protected void before() {
+ Mockito.when(ParentUtil.this.headers.getPartitionId())
+ .thenReturn(DATA_PARTITION);
+ ParentUtil.this.mongoTemplateHelper.dropCollections();
+ PartitionInfoAws partitionInfoAws = new PartitionInfoAws();
+ Property tenantIdProperty = new Property();
+ tenantIdProperty.setValue(DATA_PARTITION);
+ partitionInfoAws.setTenantIdProperty(tenantIdProperty);
+ Mockito.when(partitionServiceClient.getPartition(anyString())).thenReturn(partitionInfoAws);
+ }
+
+ @Override
+ protected void after() {
+ ParentUtil.this.mongoTemplateHelper.dropCollections();
+ }
+ };
+
+ @Autowired
+ public void set(MongoTemplate mongoTemplate) {
+ this.mongoTemplateHelper = new MongoTemplateHelper(mongoTemplate);
+ }
+}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/RecordMetadataGenerator.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/RecordMetadataGenerator.java
new file mode 100644
index 000000000..d205c91c2
--- /dev/null
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/mongo/util/RecordMetadataGenerator.java
@@ -0,0 +1,56 @@
+// Copyright MongoDB, Inc or its affiliates. All Rights Reserved.
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.aws.api.mongo.util;
+
+import org.opengroup.osdu.core.common.model.legal.Legal;
+import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
+import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.storage.provider.aws.mongo.dto.RecordMetadataMongoDBDto;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+public class RecordMetadataGenerator extends DbUtil {
+ public static RecordMetadataMongoDBDto create() {
+ return create(RECORD_ID, "default");
+ }
+
+ public static RecordMetadataMongoDBDto create(String id, String someParameter) {
+ RecordMetadata recordMetadata = new RecordMetadata();
+ Legal legal = new Legal();
+ Set<String> legalTags = new HashSet<>();
+ for (int i = 0; i < 3; i++) {
+ legalTags.add(someParameter + i);
+ }
+ legal.setLegaltags(legalTags);
+ recordMetadata.setLegal(legal);
+ recordMetadata.setKind(someParameter);
+ recordMetadata.setStatus(RecordState.active);
+ recordMetadata.setId(id);
+ return new RecordMetadataMongoDBDto(recordMetadata);
+ }
+
+ public static List<RecordMetadataMongoDBDto> generate(int count, String someParameter) {
+ List<RecordMetadataMongoDBDto> recordMetadataMongoDBDtos = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ recordMetadataMongoDBDtos.add(create(UUID.randomUUID().toString(), someParameter));
+ }
+ return recordMetadataMongoDBDtos;
+ }
+}
diff --git a/provider/storage-aws/src/test/resources/application.properties b/provider/storage-aws/src/test/resources/application.properties
new file mode 100644
index 000000000..fbe8a5280
--- /dev/null
+++ b/provider/storage-aws/src/test/resources/application.properties
@@ -0,0 +1,7 @@
+#embedded mongo
+#used for setting up necessary embedded mongo port
+spring.data.mongodb.port=27019
+
+#testing context configuration
+spring.main.allow-bean-definition-overriding=true
+repository.implementation=mongodb
\ No newline at end of file
diff --git a/testing/pom.xml b/testing/pom.xml
index 0bb7b72b6..7d2343a16 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -32,7 +32,6 @@
<modules>
<module>storage-test-core</module>
<module>storage-test-aws</module>
- <module>storage-test-aws-mongodb</module>
<module>storage-test-azure</module>
<module>storage-test-gcp</module>
<module>storage-test-ibm</module>
diff --git a/testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh b/testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh
deleted file mode 100644
index fae2267da..000000000
--- a/testing/storage-test-aws-mongodb/build-aws/prepare-dist.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright © 2020 Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script prepares the dist directory for the integration tests.
-# Must be run from the root of the repostiory
-
-# This script prepares the dist directory for the integration tests.
-# Must be run from the root of the repostiory
-
-set -e
-
-OUTPUT_DIR="${OUTPUT_DIR:-dist}"
-
-INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing/integration
-INTEGRATION_TEST_OUTPUT_BIN_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$INTEGRATION_TEST_OUTPUT_DIR}/bin
-INTEGRATION_TEST_SOURCE_DIR=testing
-INTEGRATION_TEST_SOURCE_DIR_AWS="$INTEGRATION_TEST_SOURCE_DIR"/storage-test-aws-mongodb
-INTEGRATION_TEST_SOURCE_DIR_CORE="$INTEGRATION_TEST_SOURCE_DIR"/storage-test-core
-echo "--Source directories variables--"
-echo $INTEGRATION_TEST_SOURCE_DIR_AWS
-echo $INTEGRATION_TEST_SOURCE_DIR_CORE
-echo "--Output directories variables--"
-echo $OUTPUT_DIR
-echo $INTEGRATION_TEST_OUTPUT_DIR
-echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
-
-rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
-mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
-echo "Building integration testing assemblies and gathering artifacts..."
-mvn install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
-mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
-cp "$INTEGRATION_TEST_SOURCE_DIR_AWS"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
-(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
-chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
-mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
-cp -R "$INTEGRATION_TEST_SOURCE_DIR_AWS"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/
-
diff --git a/testing/storage-test-aws-mongodb/build-aws/run-tests.sh b/testing/storage-test-aws-mongodb/build-aws/run-tests.sh
deleted file mode 100644
index b7c3b900a..000000000
--- a/testing/storage-test-aws-mongodb/build-aws/run-tests.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright © 2020 Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script prepares the dist directory for the integration tests.
-# Must be run from the root of the repostiory
-
-# This script executes the test and copies reports to the provided output directory
-# To call this script from the service working directory
-# ./dist/testing/integration/build-aws/run-tests.sh "./reports/"
-
-
-SCRIPT_SOURCE_DIR=$(dirname "$0")
-echo "Script source location"
-echo "$SCRIPT_SOURCE_DIR"
-(cd "$SCRIPT_SOURCE_DIR"/../bin && ./install-deps.sh)
-
-#### ADD REQUIRED ENVIRONMENT VARIABLES HERE ###############################################
-# The following variables are automatically populated from the environment during integration testing
-# see os-deploy-aws/build-aws/integration-test-env-variables.py for an updated list
-
-# AWS_COGNITO_CLIENT_ID
-# ELASTIC_HOST
-# ELASTIC_PORT
-# FILE_URL
-# LEGAL_URL
-# SEARCH_URL
-# STORAGE_URL
-export OTHER_RELEVANT_DATA_COUNTRIES=US
-export DEPLOY_ENV=empty
-export LEGAL_TAG=opendes-public-usa-dataset-1
-export TENANT_NAME=int-test-storage
-export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH
-export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD
-export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER
-export AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=$USER_NO_ACCESS
-export STORAGE_URL=$STORAGE_URL
-export DOMAIN=testing.com
-export LEGAL_URL=$LEGAL_URL
-
-#### RUN INTEGRATION TEST #########################################################################
-
-mvn test -f "$SCRIPT_SOURCE_DIR"/../pom.xml
-TEST_EXIT_CODE=$?
-
-#### COPY TEST REPORTS #########################################################################
-
-if [ -n "$1" ]
- then
- mkdir -p "$1"
- cp -R "$SCRIPT_SOURCE_DIR"/../target/surefire-reports "$1"
-fi
-
-exit $TEST_EXIT_CODE
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/pom.xml b/testing/storage-test-aws-mongodb/pom.xml
deleted file mode 100644
index eee538e62..000000000
--- a/testing/storage-test-aws-mongodb/pom.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Copyright © 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>os-storage-testing</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- <version>0.16.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.opengroup.osdu.storage</groupId>
- <artifactId>storage-test-aws-mongodb</artifactId>
- <version>0.16.0-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <properties>
- <maven.compiler.target>1.8</maven.compiler.target>
- <maven.compiler.source>1.8</maven.compiler.source>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.main.basedir>${project.basedir}</project.main.basedir>
- </properties>
- <dependencies>
- <!-- Internal packages -->
- <dependency>
- <groupId>org.opengroup.osdu.core.aws</groupId>
- <artifactId>os-core-lib-aws</artifactId>
- <version>0.3.8</version>
- </dependency>
-
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common</artifactId>
- <version>0.13.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.opengroup.osdu.storage</groupId>
- <artifactId>storage-test-core</artifactId>
- <version>0.16.0-SNAPSHOT</version>
- </dependency>
-
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-cognitoidp</artifactId>
- <version>1.11.676</version>
- </dependency>
-
- <!-- third party Apache 2.0 license packages -->
- <dependency>
- <groupId>javax.json</groupId>
- <artifactId>javax.json-api</artifactId>
- <version>1.1.2</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish</groupId>
- <artifactId>javax.json</artifactId>
- <version>1.1.2</version>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- <version>1.19.4</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.5</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson-databind.version}</version>
- </dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt</artifactId>
- <version>0.9.1</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.2</version>
- <scope>runtime</scope>
- </dependency>
-
- <!-- Tests -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.5</version>
- <executions>
- <execution>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <excludes>
- <exclude>none</exclude>
- </excludes>
- <includes>
- <include>**/storage</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- <configuration>
- <trimStackTrace>false</trimStackTrace>
- <systemPropertyVariables>
- <buildDirectory>${project.build.directory}</buildDirectory>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/PubsubEndpoint/TestPubsubEndpoint.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/PubsubEndpoint/TestPubsubEndpoint.java
deleted file mode 100644
index ea5dbe6ea..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/PubsubEndpoint/TestPubsubEndpoint.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.PubsubEndpoint;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestPubsubEndpoint extends PubsubEndpointTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- PubsubEndpointTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- PubsubEndpointTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/legal/TestPopulateLegalInfoFromParentRecords.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/legal/TestPopulateLegalInfoFromParentRecords.java
deleted file mode 100644
index ec7b09970..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/legal/TestPopulateLegalInfoFromParentRecords.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.legal;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestPopulateLegalInfoFromParentRecords extends PopulateLegalInfoFromParentRecordsTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- PopulateLegalInfoFromParentRecordsTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- PopulateLegalInfoFromParentRecordsTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStorageCors.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStorageCors.java
deleted file mode 100644
index cf650f10b..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStorageCors.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.misc;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestStorageCors extends StorageCorsTests {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStress.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStress.java
deleted file mode 100644
index 6b33f5c1f..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/misc/TestStress.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.misc;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestStress extends StressTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- StressTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- StressTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryInfoIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryInfoIntegration.java
deleted file mode 100644
index 459aeffc5..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryInfoIntegration.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestGetQueryInfoIntegration extends GetQueryInfoIntegrationTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- GetQueryRecordsIntegrationTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- GetQueryRecordsIntegrationTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryKindsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryKindsIntegration.java
deleted file mode 100644
index 989aef48f..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryKindsIntegration.java
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-import org.opengroup.osdu.storage.util.SchemaUtil;
-import org.opengroup.osdu.storage.util.TenantUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class TestGetQueryKindsIntegration extends GetQueryKindsIntegrationTests {
-
- private static List<String> schemaIds;
-
- // Need at least 3 to test limit of 2
- private static final int SCHEMA_COUNT = 3;
-
- protected static final String KIND_TEMPLATE = TenantUtils.getTenantName() + ":test:testkind:1.%d." + System.currentTimeMillis();
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
-
- schemaIds = new ArrayList<>();
- for (int i = 0; i < SCHEMA_COUNT; i++) {
- String kind = String.format(KIND_TEMPLATE,i);
- SchemaUtil.create(kind, testUtils.getToken());
- schemaIds.add(kind);
- }
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- for (String kind : schemaIds) {
- SchemaUtil.delete(kind, testUtils.getToken());
- }
-
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryRecordsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryRecordsIntegration.java
deleted file mode 100644
index ac1c6a8f1..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestGetQueryRecordsIntegration.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestGetQueryRecordsIntegration extends GetQueryRecordsIntegrationTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- GetQueryRecordsIntegrationTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- GetQueryRecordsIntegrationTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostFetchRecordsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostFetchRecordsIntegration.java
deleted file mode 100644
index 7fa62526c..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostFetchRecordsIntegration.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestPostFetchRecordsIntegration extends PostFetchRecordsIntegrationTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- PostFetchRecordsIntegrationTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- PostFetchRecordsIntegrationTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostQueryRecordsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostQueryRecordsIntegration.java
deleted file mode 100644
index 70c0ccf4d..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestPostQueryRecordsIntegration.java
+++ /dev/null
@@ -1,49 +0,0 @@
-//Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestPostQueryRecordsIntegration extends PostQueryRecordsIntegrationTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- PostQueryRecordsIntegrationTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- PostQueryRecordsIntegrationTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestStorageQuerySuccessful.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestStorageQuerySuccessful.java
deleted file mode 100644
index 8801cf257..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/query/TestStorageQuerySuccessful.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.query;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-import org.opengroup.osdu.storage.util.SchemaUtil;
-import org.opengroup.osdu.storage.util.TenantUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class TestStorageQuerySuccessful extends StorageQuerySuccessfulTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- private static List<String> schemaIds;
-
- // Need at least 11 to require a cursor when querying for 10 items
- private static final int SCHEMA_COUNT = 11;
-
- protected static final String KIND_TEMPLATE = TenantUtils.getTenantName() + ":test:testkind:1.%d." + System.currentTimeMillis();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- StorageQuerySuccessfulTest.classSetup(awsTestUtils.getToken());
-
- schemaIds = new ArrayList<>();
- for (int i = 0; i < SCHEMA_COUNT; i++) {
- String kind = String.format(KIND_TEMPLATE,i);
- SchemaUtil.create(kind, awsTestUtils.getToken());
- schemaIds.add(kind);
- }
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- for (String kind : schemaIds) {
- SchemaUtil.delete(kind, awsTestUtils.getToken());
- }
-
- StorageQuerySuccessfulTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestDeleteRecordLogicallyAndItsVersions.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestDeleteRecordLogicallyAndItsVersions.java
deleted file mode 100644
index 467ccf171..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestDeleteRecordLogicallyAndItsVersions.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestDeleteRecordLogicallyAndItsVersions extends DeleteRecordLogicallyAndItsVersionsTest {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- super.setup();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- super.tearDown();
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestGetRecordsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestGetRecordsIntegration.java
deleted file mode 100644
index a81f417fe..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestGetRecordsIntegration.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestGetRecordsIntegration extends GetRecordsIntegrationTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- GetRecordsIntegrationTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- GetRecordsIntegrationTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestIngestRecordNotFound.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestIngestRecordNotFound.java
deleted file mode 100644
index 7a20b9f38..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestIngestRecordNotFound.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import com.sun.jersey.api.client.ClientResponse;
-
-import org.apache.http.HttpStatus;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-import org.opengroup.osdu.storage.util.HeaderUtils;
-import org.opengroup.osdu.storage.util.RecordUtil;
-import org.opengroup.osdu.storage.util.TenantUtils;
-import org.opengroup.osdu.storage.util.TestUtils;
-
-public class TestIngestRecordNotFound extends IngestRecordNotFoundTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- IngestRecordNotFoundTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- IngestRecordNotFoundTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
- @Override
- @Test
- public void should_returnBadRequest_when_userGroupDoesNotExist() throws Exception {
-
- String group = String.format("data.thisDataGrpDoesNotExsist@%s", TestUtils.getAclSuffix());
-
- String record = RecordUtil.createDefaultJsonRecord(RECORD_ID, KIND, LEGAL_TAG).replace(TestUtils.getAcl(), group);
- ClientResponse response = TestUtils.send("records", "PUT", HeaderUtils.getHeaders(TenantUtils.getTenantName(), testUtils.getToken()), record, "");
- // it's a much simpler implementation to just check if the user is in the group that is being saved and if not to skip
- // per previous integration test requirements
- TestUtils.getResult(response, HttpStatus.SC_FORBIDDEN, String.class);
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestLogicalRecordDelete.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestLogicalRecordDelete.java
deleted file mode 100644
index ec4230ea2..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestLogicalRecordDelete.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestLogicalRecordDelete extends LogicalRecordDeleteTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- LogicalRecordDeleteTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- LogicalRecordDeleteTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestPurgeRecordsIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestPurgeRecordsIntegration.java
deleted file mode 100644
index 930a51534..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestPurgeRecordsIntegration.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestPurgeRecordsIntegration extends PurgeRecordsIntegrationTest {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- PurgeRecordsIntegrationTest.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- PurgeRecordsIntegrationTest.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java
deleted file mode 100644
index 4301cfe12..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordAccessAuthorization.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestRecordAccessAuthorization extends RecordAccessAuthorizationTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- RecordAccessAuthorizationTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- RecordAccessAuthorizationTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordWithNullField.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordWithNullField.java
deleted file mode 100644
index eeff9f58e..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordWithNullField.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestRecordWithNullField extends RecordWithNullFieldTest {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- super.setup();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- super.tearDown();
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java
deleted file mode 100644
index 2d26a101e..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestRecordsApiAcceptance.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestRecordsApiAcceptance extends RecordsApiAcceptanceTests {
-
- private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
-
- @BeforeClass
- public static void classSetup() throws Exception {
- RecordsApiAcceptanceTests.classSetup(awsTestUtils.getToken());
- }
-
- @AfterClass
- public static void classTearDown() throws Exception {
- RecordsApiAcceptanceTests.classTearDown(awsTestUtils.getToken());
- }
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestUpdateRecordsMetadata.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestUpdateRecordsMetadata.java
deleted file mode 100644
index 57c0138a3..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/records/TestUpdateRecordsMetadata.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.records;
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestUpdateRecordsMetadata extends UpdateRecordsMetadataTest {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- super.setup();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- super.tearDown();
- this.testUtils = null;
- }
-
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestCreateSchemaIntegration.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestCreateSchemaIntegration.java
deleted file mode 100644
index e3ef90824..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestCreateSchemaIntegration.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.schema;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestCreateSchemaIntegration extends CreateSchemaIntegrationTests {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
\ No newline at end of file
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestStorageSchemaNegative.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestStorageSchemaNegative.java
deleted file mode 100644
index aec563422..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/schema/TestStorageSchemaNegative.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.schema;
-
-import org.junit.After;
-import org.junit.Before;
-import org.opengroup.osdu.storage.util.AWSTestUtils;
-
-public class TestStorageSchemaNegative extends StorageSchemaNegativeTest {
-
- @Before
- @Override
- public void setup() throws Exception {
- this.testUtils = new AWSTestUtils();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- this.testUtils = null;
- }
-
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoBuilder.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoBuilder.java
deleted file mode 100644
index 77081605d..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoBuilder.java
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package org.opengroup.osdu.storage.util;
-
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider;
-import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProviderClientBuilder;
-
-public class AWSCognitoBuilder {
- public static AWSCognitoIdentityProvider generateCognitoClient(){
- return AWSCognitoIdentityProviderClientBuilder.standard()
- .withCredentials(new EnvironmentVariableCredentialsProvider())
- .build();
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoClient.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoClient.java
deleted file mode 100644
index 955766eea..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSCognitoClient.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package org.opengroup.osdu.storage.util;
-
-import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider;
-import com.amazonaws.services.cognitoidp.model.InitiateAuthRequest;
-import com.amazonaws.services.cognitoidp.model.InitiateAuthResult;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class AWSCognitoClient {
- public static String getTokenForUserWithAccess(){
- String clientId = System.getProperty("AWS_COGNITO_CLIENT_ID", System.getenv("AWS_COGNITO_CLIENT_ID"));
- String authFlow = System.getProperty("AWS_COGNITO_AUTH_FLOW", System.getenv("AWS_COGNITO_AUTH_FLOW"));
- Map<String, String> authParameters = new HashMap<>();
- authParameters.put("USERNAME", System.getProperty("AWS_COGNITO_AUTH_PARAMS_USER", System.getenv("AWS_COGNITO_AUTH_PARAMS_USER")));
- authParameters.put("PASSWORD", System.getProperty("AWS_COGNITO_AUTH_PARAMS_PASSWORD", System.getenv("AWS_COGNITO_AUTH_PARAMS_PASSWORD")));
-
- AWSCognitoIdentityProvider provider = AWSCognitoBuilder.generateCognitoClient();
- InitiateAuthRequest request = new InitiateAuthRequest();
- request.setClientId(clientId);
- request.setAuthFlow(authFlow);
- request.setAuthParameters(authParameters);
-
- InitiateAuthResult result = provider.initiateAuth(request);
- return result.getAuthenticationResult().getAccessToken();
- }
-
- public static String getTokenForUserWithNoAccess(){
- String clientId = System.getProperty("AWS_COGNITO_CLIENT_ID", System.getenv("AWS_COGNITO_CLIENT_ID"));
- String authFlow = System.getProperty("AWS_COGNITO_AUTH_FLOW", System.getenv("AWS_COGNITO_AUTH_FLOW"));
- Map<String, String> authParameters = new HashMap<>();
- authParameters.put("USERNAME", System.getProperty("AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS", System.getenv("AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS")));
- authParameters.put("PASSWORD", System.getProperty("AWS_COGNITO_AUTH_PARAMS_PASSWORD", System.getenv("AWS_COGNITO_AUTH_PARAMS_PASSWORD")));
-
- AWSCognitoIdentityProvider provider = AWSCognitoBuilder.generateCognitoClient();
- InitiateAuthRequest request = new InitiateAuthRequest();
- request.setClientId(clientId);
- request.setAuthFlow(authFlow);
- request.setAuthParameters(authParameters);
-
- InitiateAuthResult result = provider.initiateAuth(request);
- return result.getAuthenticationResult().getAccessToken();
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSTestUtils.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSTestUtils.java
deleted file mode 100644
index d57550ad2..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/AWSTestUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.util;
-
-import com.google.common.base.Strings;
-import org.opengroup.osdu.core.aws.cognito.AWSCognitoClient;
-
-public class AWSTestUtils extends TestUtils {
- private static String token;
- private static String noDataAccesstoken;
- private static AWSCognitoClient awsCognitoClient = null;
-
- @Override
- public synchronized String getToken() throws Exception {
- if (Strings.isNullOrEmpty(token)) {
- token = getAwsCognitoClient().getTokenForUserWithAccess();
- }
- return "Bearer " + token;
- }
-
- @Override
- public synchronized String getNoDataAccessToken() throws Exception {
- if (Strings.isNullOrEmpty(noDataAccesstoken)) {
- noDataAccesstoken = getAwsCognitoClient().getTokenForUserWithNoAccess();
- }
- return "Bearer " + noDataAccesstoken;
- }
-
- private AWSCognitoClient getAwsCognitoClient() {
- if(awsCognitoClient == null)
- awsCognitoClient = new AWSCognitoClient();
- return awsCognitoClient;
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/SchemaUtil.java b/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/SchemaUtil.java
deleted file mode 100644
index c0038e14e..000000000
--- a/testing/storage-test-aws-mongodb/src/test/java/org/opengroup/osdu/storage/util/SchemaUtil.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright © 2020 Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package org.opengroup.osdu.storage.util;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.sun.jersey.api.client.ClientResponse;
-
-public class SchemaUtil {
-
- protected final String schema = TenantUtils.getTenantName() + ":storage:inttest:1.0.0"
- + System.currentTimeMillis();
-
- public static ClientResponse create(String kind, String token) throws Exception {
- return TestUtils.send("schemas", "POST", HeaderUtils.getHeaders(TenantUtils.getTenantName(), token), SchemaUtil.validSchemaPostBody(kind), "");
- }
-
- public static ClientResponse delete(String kind, String token) throws Exception {
- return TestUtils.send("schemas/" + kind, "DELETE", HeaderUtils.getHeaders(TenantUtils.getTenantName(), token), "", "");
- }
-
- protected static String validSchemaPostBody(String kind) {
-
- JsonObject item1Ext = new JsonObject();
- item1Ext.addProperty("indexerTip", "call911");
-
- JsonObject item1 = new JsonObject();
- item1.addProperty("path", "name");
- item1.addProperty("kind", "string");
- item1.add("ext", item1Ext);
-
- JsonObject item2Ext = new JsonObject();
- item2Ext.addProperty("address.city", "this is a weird string");
- item2Ext.addProperty("address.country", "country with two letters");
-
- JsonObject item2 = new JsonObject();
- item2.addProperty("path", "age");
- item2.addProperty("kind", "int");
- item2.add("ext", item2Ext);
-
- JsonArray schemaItems = new JsonArray();
- schemaItems.add(item1);
- schemaItems.add(item2);
-
- JsonObject schema = new JsonObject();
- schema.addProperty("kind", kind);
- schema.add("schema", schemaItems);
- schema.add("ext", item2Ext);
-
- return schema.toString();
- }
-}
diff --git a/testing/storage-test-aws-mongodb/src/test/resources/logback-test.xml b/testing/storage-test-aws-mongodb/src/test/resources/logback-test.xml
deleted file mode 100644
index dafd8ae39..000000000
--- a/testing/storage-test-aws-mongodb/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <include resource="org/springframework/boot/logging/logback/base.xml" />
- <root level="INFO" />
- <logger name="org.springframework" level="INFO"/>
-</configuration>
\ No newline at end of file
--
GitLab