Multi-tenancy

commit 57a24b40 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 13:01:23 GMT-0500 (Central Daylight Time) 

    Consolidating s3 client classes


commit c6f635ba 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 12:40:15 GMT-0500 (Central Daylight Time) 

    Cleaning up


commit 5c9e8475 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 11:59:24 GMT-0500 (Central Daylight Time) 

    Updating core lib aws version


commit e3e0f9c4 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 11:45:09 GMT-0500 (Central Daylight Time) 

    Adjusting  tests for mult-tenancy


commit 7064328e 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 10:57:11 GMT-0500 (Central Daylight Time) 

    Deleted accidental key


commit a65471ea 
Author: Spencer Sutton <spencerpsutton@yahoo.com> 
Date: Wed May 05 2021 10:56:30 GMT-0500 (Central Daylight Time) 

    Updating to be multi-tenant

provider/legal-aws/pom.xml

@@ -37,7 +37,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.17</version>
+            <version>0.9.1-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>

provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/countries/StorageReaderFactoryImpl.java

@@ -16,36 +16,23 @@ package org.opengroup.osdu.legal.aws.countries;
 
 import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
 import org.opengroup.osdu.core.aws.ssm.SSMConfig;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
 import org.opengroup.osdu.legal.provider.interfaces.IStorageReader;
 import org.opengroup.osdu.legal.provider.interfaces.IStorageReaderFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
+import javax.inject.Inject;
+
 @Component
 public class StorageReaderFactoryImpl implements IStorageReaderFactory {
 
-    private String legalConfigBucketName;
-
-    @Value("${aws.s3.legal.config.file-name}")
-    private String legalConfigFileName;
-
-    @Value("${aws.s3.endpoint}")
-    private String awsS3Endpoint;
-
-    @Value("${aws.s3.region}")
-    private String awsS3Region;
-
-    @Value("${aws.legal.s3.bucket.name}")
-    String parameter;
-
-    private ParameterStorePropertySource ssm;
+    @Inject
+    StorageReaderImpl storageReaderImpl;
 
     @Override
     public IStorageReader getReader(TenantInfo tenant, String projectRegion) {
-        SSMConfig ssmConfig = new SSMConfig();
-        ssm = ssmConfig.amazonSSM();
-        legalConfigBucketName = ssm.getProperty(parameter).toString();
-        return new StorageReaderImpl(tenant, legalConfigBucketName, legalConfigFileName, awsS3Endpoint, awsS3Region);
+        return storageReaderImpl;
     }
 }

provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/countries/StorageReaderImpl.java

@@ -14,22 +14,55 @@
 
 package org.opengroup.osdu.legal.aws.countries;
 
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import org.opengroup.osdu.core.aws.s3.IS3ClientFactory;
+import org.opengroup.osdu.core.aws.s3.S3ClientFactory;
+import org.opengroup.osdu.core.aws.s3.S3ClientWithBucket;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.legal.aws.tags.dataaccess.S3RecordClient;
 import org.opengroup.osdu.legal.provider.interfaces.IStorageReader;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 
+import javax.inject.Inject;
+
+@Component
 public class StorageReaderImpl implements IStorageReader {
-    private final TenantInfo tenantInfo;
-    private S3RecordClient client;
 
-    public StorageReaderImpl(TenantInfo tenantInfo, String legalConfigBucketName, String legalConfigFileName,
-                             String awsS3Endpoint, String awsS3Region) {
-        this.tenantInfo = tenantInfo;
-        client = new S3RecordClient(legalConfigBucketName, legalConfigFileName, awsS3Endpoint, awsS3Region);
-    }
+    @Inject
+    private DpsHeaders dpsHeaders;
+
+    @Inject
+    private IS3ClientFactory s3ClientFactory;
+
+    @Value("${aws.s3.configbucket.ssm.relativePath}")
+    private String s3ConfigBucketParameterRelativePath;
+
+    @Value("${aws.s3.legal.config.file-name}")
+    private String legalConfigFileName;
 
     @Override
     public byte[] readAllBytes() {
-        return (client.getConfigFile().getBytes()); //should return a json format of an array of Country class
+        return getConfigFile().getBytes(); //should return a json format of an array of Country class
+    }
+
+    public String getConfigFile(){
+        S3ClientWithBucket s3ClientWithBucket = s3ClientFactory.getS3ClientForPartition(
+                dpsHeaders.getPartitionId(), s3ConfigBucketParameterRelativePath);
+        AmazonS3 s3 = s3ClientWithBucket.getS3Client();
+        String legalConfigBucketName = s3ClientWithBucket.getBucketName();
+
+        String contents = "";
+        try {
+            contents = s3.getObjectAsString(legalConfigBucketName, legalConfigFileName);
+        } catch (AmazonServiceException e) {
+            throw new AppException(500, "Amazon service exception getting config file", e.getMessage());
+        } catch (SdkClientException e) {
+            throw new AppException(500, "Amazon Sdk client exception getting config file", e.getMessage());
+        }
+        return contents;
     }
 }

provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/tags/dataaccess/LegalDoc.java

@@ -24,6 +24,9 @@ public class LegalDoc {
     @DynamoDBHashKey(attributeName = "Id")
     private String id;
 
+    @DynamoDBRangeKey(attributeName = "dataPartitionId")
+    private String dataPartitionId;
+
     @DynamoDBAttribute(attributeName = "Name")
     private String name;
 
@@ -46,6 +49,14 @@ public class LegalDoc {
         this.id = id;
     }
 
+    public String getDataPartitionId(){
+        return dataPartitionId;
+    }
+
+    public void setDataPartitionId(String tenant){
+        this.dataPartitionId = tenant;
+    }
+
     public void setName(String name) {
         this.name = name;
     }

provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/tags/dataaccess/LegalTagRepositoryImpl.java

@@ -14,9 +14,14 @@
 
 package org.opengroup.osdu.legal.aws.tags.dataaccess;
 
+import com.amazonaws.services.dynamodbv2.model.AttributeValue;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpStatus;
 import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
+import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
+import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
 import org.opengroup.osdu.core.aws.dynamodb.QueryPageResult;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.legal.ListLegalTagArgs;
 import org.opengroup.osdu.core.common.model.legal.LegalTag;
 
@@ -26,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 
 import javax.annotation.PostConstruct;
+import javax.inject.Inject;
 import java.io.UnsupportedEncodingException;
 import java.util.*;
 
@@ -41,15 +47,24 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
     @Value("${aws.dynamodb.endpoint}")
     String dynamoDbEndpoint;
 
-    private DynamoDBQueryHelper queryHelper;
+    @Inject
+    private DpsHeaders headers;
 
-    @PostConstruct
-    public void init(){
-        queryHelper = new DynamoDBQueryHelper(dynamoDbEndpoint, dynamoDbRegion, tablePrefix);
+    @Inject
+    private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
+
+    @Value("${aws.dynamodb.legalTable.ssm.relativePath}")
+    String legalRepositoryTableParameterRelativePath;
+
+    private DynamoDBQueryHelperV2 getLegalRepositoryQueryHelper() {
+        return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, legalRepositoryTableParameterRelativePath);
     }
 
+
     @Override
     public Long create(LegalTag legalTag) {
+        DynamoDBQueryHelperV2 queryHelper = getLegalRepositoryQueryHelper();
+
         LegalDoc legalDoc = CreateLegalDocFromTag(legalTag);
         if (queryHelper.keyExistsInTable(LegalDoc.class, legalDoc)){
             throw new AppException(409, "Legal tag conflict", String.format(
@@ -61,10 +76,12 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
 
     @Override
     public Collection<LegalTag> get(long[] ids) {
+        DynamoDBQueryHelperV2 queryHelper = getLegalRepositoryQueryHelper();
+
         List<LegalTag> tags = new ArrayList<>();
 
         for(long id: ids) {
-            LegalDoc ld = queryHelper.loadByPrimaryKey(LegalDoc.class, String.valueOf(id)); //dynamoDBLegal.findById(String.valueOf(id));
+            LegalDoc ld = queryHelper.loadByPrimaryKey(LegalDoc.class, String.valueOf(id), headers.getPartitionId()); //dynamoDBLegal.findById(String.valueOf(id));
             if(ld != null) {
                 tags.add(CreateLegalTagFromDoc(ld));
             }
@@ -75,9 +92,11 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
 
     @Override
     public Boolean delete(LegalTag legalTag) {
+        DynamoDBQueryHelperV2 queryHelper = getLegalRepositoryQueryHelper();
+
         Boolean result = true;
         try {
-            queryHelper.deleteByPrimaryKey(LegalDoc.class, String.valueOf(legalTag.getId()));
+            queryHelper.deleteByPrimaryKey(LegalDoc.class, String.valueOf(legalTag.getId()), headers.getPartitionId());
         } catch (Exception e){ // should be dynamodb specific exception
             result = false;
             // might need to throw app exception
@@ -94,15 +113,24 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
 
     @Override
     public Collection<LegalTag> list(ListLegalTagArgs args) {
+        DynamoDBQueryHelperV2 queryHelper = getLegalRepositoryQueryHelper();
+
+        String filterExpression = "dataPartitionId = :partitionId";
+
+        AttributeValue dataPartitionAttributeValue = new AttributeValue(headers.getPartitionId());
+
+        Map<String, AttributeValue> eav = new HashMap<>();
+        eav.put(":partitionId", dataPartitionAttributeValue);
+
         int limit = args.getLimit();
 
         List<LegalDoc> docs = null;
         List<LegalTag> tags = new ArrayList<>();
         try {
             if(limit <= 0){
-                docs = queryHelper.scanTable(LegalDoc.class);
+                docs = queryHelper.scanTable(LegalDoc.class, filterExpression, eav);
             } else {
-                QueryPageResult<LegalDoc> scanPageResults = queryHelper.scanPage(LegalDoc.class, limit, args.getCursor());
+                QueryPageResult<LegalDoc> scanPageResults = queryHelper.scanPage(LegalDoc.class, limit, args.getCursor(),filterExpression, eav);
                 if (scanPageResults != null) docs = scanPageResults.results;
             }
         } catch (UnsupportedEncodingException e) {
@@ -120,6 +148,8 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
     }
 
     private Long save(LegalDoc legalDoc){
+        DynamoDBQueryHelperV2 queryHelper = getLegalRepositoryQueryHelper();
+
         Long id = -1L;
         if(legalDoc != null){
             queryHelper.save(legalDoc);
@@ -141,6 +171,7 @@ public class LegalTagRepositoryImpl implements ILegalTagRepository {
     private LegalDoc CreateLegalDocFromTag(LegalTag legalTag){
         LegalDoc legalDoc = new LegalDoc();
         legalDoc.setId(String.valueOf(legalTag.getId()));
+        legalDoc.setDataPartitionId(headers.getPartitionId());
         legalDoc.setDescription(legalTag.getDescription());
         legalDoc.setName(legalTag.getName());
         legalDoc.setProperties(legalTag.getProperties());

provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/tags/dataaccess/S3RecordClient.java

-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.legal.aws.tags.dataaccess;
-
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.services.s3.AmazonS3;
-import org.opengroup.osdu.core.aws.s3.S3Config;
-import org.opengroup.osdu.core.common.model.http.AppException;
-
-public class S3RecordClient {
-
-    // Storing all records in one bucket does not impact performance, no need to spread keys anymore
-    private String legalConfigBucketName;
-    private String legalConfigFileName;
-
-    private AmazonS3 s3;
-
-    public S3RecordClient(String legalConfigBucketName, String legalConfigFileName, String awsS3Endpoint, String awsS3Region){
-        this.legalConfigBucketName = legalConfigBucketName;
-        this.legalConfigFileName = legalConfigFileName;
-        S3Config s3Config = new S3Config(awsS3Endpoint, awsS3Region);
-        s3 = s3Config.amazonS3();
-    }
-
-    public String getConfigFile(){
-        String contents = "";
-        try {
-            contents = s3.getObjectAsString(legalConfigBucketName, legalConfigFileName);
-        } catch (AmazonServiceException e) {
-            throw new AppException(500, "Amazon service exception getting config file", e.getMessage());
-        } catch (SdkClientException e) {
-            throw new AppException(500, "Amazon Sdk client exception getting config file", e.getMessage());
-        }
-        return contents;
-    }
-}

provider/legal-aws/src/main/resources/application.properties

@@ -7,8 +7,14 @@ server.port=${APPLICATION_PORT}
 ## AWS Lambda configuration
 aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
 
+aws.environment=${ENVIRONMENT}
+aws.parameter.prefix=/osdu/${ENVIRONMENT}
+aws.resourcePrefix=${ENVIRONMENT}
+aws.region=${AWS_REGION}
+
 REGION=${AWS_REGION}
 AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1
+PARTITION_API=${ENTITLEMENTS_BASE_URL}/api/partition/v1
 LEGAL_HOSTNAME=notused
 CRON_JOB_IP=10.0.0.1
 ACCEPT_HTTP=true
@@ -30,7 +36,10 @@ aws.sns.region=${AWS_REGION}
 aws.ssm=${SSM_ENABLED}
 aws.ssm.prefix=/osdu/${ENVIRONMENT}
 aws.legal.sns.topic.arn=${aws.ssm.prefix}/legal/legal-sns-topic-arn
-aws.legal.s3.bucket.name=${aws.ssm.prefix}/legal/legal-s3-bucket-name
+
+aws.s3.configbucket.ssm.relativePath=${CONFIG_BUCKET_SSM_RELATIVE_PATH:legal/s3DataBucket}
+aws.s3.configfilename.ssm.relativePath=${CONFIG_FILE_SSM_RELATIVE_PATH:legal/s3ConfigFileName}
+aws.dynamodb.legalTable.ssm.relativePath=${LEGAL_REPOSITORY_SSM_RELATIVE_PATH:legal/legalTable}
 
 server.ssl.enabled=${SSL_ENABLED:true}
 server.ssl.key-store-type=PKCS12

provider/legal-aws/src/test/java/org/opengroup/osdu/legal/aws/api/LegalTagRepositoryImplTest.java

@@ -21,13 +21,17 @@ import org.junit.runner.RunWith;
 import org.mockito.*;
 import org.mockito.runners.MockitoJUnitRunner;
 import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
+import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
+import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
 import org.opengroup.osdu.core.aws.dynamodb.QueryPageResult;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.legal.aws.tags.dataaccess.LegalDoc;
 import org.opengroup.osdu.legal.aws.tags.dataaccess.LegalTagRepositoryImpl;
 import org.opengroup.osdu.core.common.model.legal.ListLegalTagArgs;
 import org.opengroup.osdu.core.common.model.legal.LegalTag;
 import org.springframework.boot.test.context.SpringBootTest;
 
+import javax.inject.Inject;
 import java.io.UnsupportedEncodingException;
 import java.util.*;
 
@@ -41,11 +45,19 @@ public class LegalTagRepositoryImplTest {
     private LegalTagRepositoryImpl repo = new LegalTagRepositoryImpl();
 
     @Mock
-    DynamoDBQueryHelper queryHelper;
+    private DynamoDBQueryHelperV2 queryHelper;
+
+    @Mock
+    private DpsHeaders headers;
+
+    @Mock
+    private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
 
     @Before
     public void setUp() {
         initMocks(this);
+        Mockito.when(dynamoDBQueryHelperFactory.getQueryHelperForPartition(Mockito.any(DpsHeaders.class), Mockito.any()))
+                .thenReturn(queryHelper);
     }
 
     @Test
@@ -72,15 +84,13 @@ public class LegalTagRepositoryImplTest {
             1L, 2L, 3L
         };
 
-        Mockito.when(queryHelper.loadByPrimaryKey(Mockito.any(), Mockito.anyString())).thenReturn(null);
-
         // act
         repo.get(ids);
 
         // assert
-        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("1"));
-        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("2"));
-        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("3"));
+        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("1"), Mockito.any());
+        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("2"), Mockito.any());
+        Mockito.verify(queryHelper, Mockito.times(1)).loadByPrimaryKey(Mockito.any(), Mockito.eq("3"), Mockito.any());
     }
 
     @Test
@@ -96,7 +106,7 @@ public class LegalTagRepositoryImplTest {
         repo.delete(tag);
 
         // assert
-        Mockito.verify(queryHelper, Mockito.times(1)).deleteByPrimaryKey(Mockito.any(), Mockito.eq("-100"));
+        Mockito.verify(queryHelper, Mockito.times(1)).deleteByPrimaryKey(Mockito.any(), Mockito.eq("-100"), Mockito.any());
     }
 
     @Test
@@ -133,7 +143,7 @@ public class LegalTagRepositoryImplTest {
         QueryPageResult<LegalDoc> page = new QueryPageResult<LegalDoc>(null, listLd);
 
         try {
-            Mockito.when(queryHelper.scanPage(Mockito.eq(LegalDoc.class), Mockito.eq(100),Mockito.any()))
+            Mockito.when(queryHelper.scanPage(Mockito.eq(LegalDoc.class), Mockito.eq(100),Mockito.any(), Mockito.any(), Mockito.any()))
                     .thenReturn(page);
         } catch (UnsupportedEncodingException e) {
             Assert.fail("Unexpected exception");
@@ -163,13 +173,6 @@ public class LegalTagRepositoryImplTest {
         listLd.add(ld);
         QueryPageResult<LegalDoc> page = new QueryPageResult<LegalDoc>(null, listLd);
 
-        try {
-            Mockito.when(queryHelper.scanPage(Mockito.eq(LegalDoc.class), Mockito.eq(100),Mockito.any()))
-                    .thenReturn(page);
-        } catch (UnsupportedEncodingException e) {
-            Assert.fail("Unexpected exception");
-        }
-
         // act
         ArrayList<LegalTag> tags = (ArrayList)repo.list(args);
 
@@ -185,13 +188,6 @@ public class LegalTagRepositoryImplTest {
         args.setCursor("{{\"title\"={\"S\":\"Monster on the Campus\",}},{\"title\":{\"S\":\"+1\",}},}}}");
         args.setLimit(100);
 
-        try {
-            Mockito.when(queryHelper.scanPage(Mockito.eq(LegalDoc.class), Mockito.eq(100),Mockito.any()))
-                    .thenReturn(null);
-        } catch (UnsupportedEncodingException e) {
-            Assert.fail("Unexpected exception");
-        }
-
         // act
         ArrayList<LegalTag> tags = (ArrayList)repo.list(args);