Merge branch 'aws-multitenant-v2' into 'master'

Bug fix for list groups, update to latest os-core-lib-aws  and update bootstrapping groups

See merge request !77

devops/aws/bootstrap.sh

@@ -29,4 +29,4 @@ fi
 export APP_KEY=""
 export DATA_PARTITION=opendes
 
-python $AWS_DEPLOYMENTS_SUBDIR/initTenant.py -u $AWS_ENTITLEMENTSV2_SERVICE_URL -t $DATA_PARTITION
+python $AWS_DEPLOYMENTS_SUBDIR/initTenant.py -u $AWS_ENTITLEMENTSV2_SERVICE_URL -t $DATA_PARTITION -p $AWS_DEPLOYMENTS_SUBDIR

devops/aws/groups_integration_test_limited_access_user_opendes.json

+{
+  "Users": ["users","service.entitlements.user"],
+  "Storage":["service.storage.admin"]
+}

devops/aws/groups_integration_test_user_common.json

+{
+  "Users": ["users","service.entitlements.user"],
+  "Indexer":["service.indexer.admin","service.indexer.creator","service.indexer.viewer"],
+  "Schema":["service.schema-service.editors","service.schema-service.viewers"]
+  
+}

devops/aws/groups_integration_test_user_opendes.json

+{
+  "Users": ["users","users.data.root","users.datalake.ops","users.datalake.admins","users.datalake.editors","users.datalake.viewers"],
+  "Entitlements":["service.entitlements.admin","service.entitlements.user"],
+  "Storage": ["service.storage.admin","service.storage.creator","service.storage.viewer","data.test1","data.integration.test"],
+  "Search":["service.search.admin","service.search.user"],
+  "Legal":["service.legal.admin","service.legal.editor","service.legal.user"],
+  "Indexer":["service.indexer.admin","service.indexer.creator","service.indexer.viewer"],
+  "Schema":["service.schema-service.editors","service.schema-service.viewers"],
+  "Unit": ["service.unit.admin","service.unit.creator","service.unit.viewer"],
+  "DataWorkflow":["service.workflow.admin","service.workflow.creator","service.workflow.viewer"],
+  "Ingestion":["service.ingest.admin","service.ingest.creator","service.ingest.viewer"],
+  "SeismicStore":["service.seismic-store.admin","service.seismic-store.creator","service.seismic-store.viewer"],
+  "BinaryDMS":["service.binarydms.admin","service.binarydms.creator","service.binarydms.viewer"],
+  "CSVParser":["service.csv-parser.admin","service.csv-parser.creator","service.csv-parser.viewer"],
+  "EDSDMS":["service.edsdms.admin","service.edsdms.creator","service.edsdms.viewer"],
+  "File":["service.file.editors","service.file.viewers"],
+  "Policy":["service.policy.admin","service.policy.creator","service.policy.viewer"],
+  "Other":["cron.job"]   
+}

devops/aws/initTenant.py

@@ -14,23 +14,56 @@
 import argparse
 import os
 import requests
+import json
+
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-u', help='The complete URL to the Entitlements V2 Service.', default=None)
-parser.add_argument('-t', help='The tenant for which groups are being provisioned.', default=None)
+parser.add_argument('-t', help='The tenant for which groups are being provisioned.', default='opendes')
+parser.add_argument('-c', help='The common data partition name for which groups are being provisioned.', default='common')
+parser.add_argument('-p', help='Path to where the groups json is stored.', default=None)
+parser.add_argument('-i', help='Integration test user email', default='admin@testing.com')
+parser.add_argument('-l', help='Integration test limited access user email', default='noaccess@testing.com')
+parser.add_argument('-d', help='Groups domain', default='contoso.com')
 arguments = parser.parse_args()
 if arguments.u is not None:
     url = arguments.u
 if arguments.t is not None:
     tenant = arguments.t
+if arguments.c is not None:
+    common_tenant = arguments.c
+if arguments.p is not None:
+    groups_json_path = arguments.p
+if arguments.i is not None:
+    integration_test_user=arguments.i
+if arguments.l is not None:
+    integration_test_limited_access_user=arguments.l
+if arguments.d is not None:
+    group_domain=arguments.d
 token = os.environ.get('BEARER_TOKEN')
 initTenantUrl=url+'tenant-provisioning'
 
-#call init API to provision groups for Service Principal
+#call init API to provision groups for Service Principal for opendes tenant
+print(token)
+print(initTenantUrl)
+headers = {
+            'data-partition-id': tenant,
+            'Content-Type': 'application/json',
+            'Authorization': token
+        }
+print(headers)
+method = 'POST'
+response = requests.request(method,initTenantUrl, headers=headers)
+print(response.status_code)
+if response.status_code==200:
+    print('The Entitlements V2 bootstrapping for ServicePrincipal successful for tenant:'+tenant)
+else:
+    print('The Entitlements V2 bootstrapping for ServicePrincipal failed for tenant:'+tenant)
 
+#call init API to provision groups for Service Principal for common tenant
 
 headers = {
-            'data-partition-id': tenant,
+            'data-partition-id': common_tenant,
             'Content-Type': 'application/json',
             'Authorization': token
         }
@@ -38,7 +71,84 @@ method = 'POST'
 response = requests.request(method,initTenantUrl, headers=headers)
 print(response.status_code)
 if response.status_code==200:
-    print('The Entitlements V2 bootstrapping successful for tenant:'+tenant)
+    print('The Entitlements V2 bootstrapping for ServicePrincipal successful for tenant:'+common_tenant)
 else:
-    print('The Entitlements V2 bootstrapping failed for tenant:'+tenant)
+    print('The Entitlements V2 bootstrapping for ServicePrincipal failed for tenant:'+common_tenant)
+
+
+#call Add Member API to provision groups for Integration test user for opendes partition
+headers = {
+            'data-partition-id': tenant,
+            'Content-Type': 'application/json',
+            'Authorization': token
+        }
+method = 'POST'
+data = {'email': integration_test_user, 'role':'MEMBER'}
+f = open(groups_json_path+'/groups_integration_test_user_opendes.json')
+groups_data = json.load(f)
+f.close()
+
+for i in groups_data:
+    arr = groups_data[i]
+    for group_name in arr:
+        group_email= group_name+'@'+tenant+'.'+group_domain
+        addMemberUrl=url+'groups/'+group_email+'/members'
+        response = requests.request(method,addMemberUrl, headers=headers, data=json.dumps(data))
+        if response.status_code==200:
+            print('The Entitlements V2 bootstrapping for Integration test user successful for group_email:'+group_email)
+        elif response.status_code==409:
+                print('The Entitlements V2 group for Integration test user already exists for group_email:'+group_email)
+        else:
+            print('The Entitlements V2 bootstrapping for Integration test user failed for group_email:'+group_email+'Error response code: '+str(response.status_code))
+
+
+#call Add Member API to provision groups for Integration test user for common data partition
+headers = {
+            'data-partition-id': common_tenant,
+            'Content-Type': 'application/json',
+            'Authorization': token
+        }
+method = 'POST'
+data = {'email': integration_test_user, 'role':'MEMBER'}
+f = open(groups_json_path+'/groups_integration_test_user_common.json')
+groups_data = json.load(f)
+f.close()
+
+for i in groups_data:
+    arr = groups_data[i]
+    for group_name in arr:
+        group_email= group_name+'@'+common_tenant+'.'+group_domain
+        addMemberUrl=url+'groups/'+group_email+'/members'
+        response = requests.request(method,addMemberUrl, headers=headers, data=json.dumps(data))
+        if response.status_code==200:
+            print('The Entitlements V2 bootstrapping for Integration test user successful for group_email:'+group_email)
+        elif response.status_code==409:
+                print('The Entitlements V2 group for Integration test user already exists for group_email:'+group_email)
+        else:
+            print('The Entitlements V2 bootstrapping for Integration test user failed for group_email:'+group_email+'Error response code: '+str(response.status_code))
+
+
+#call Add Member API to provision groups for Integration test limited access user
+headers = {
+            'data-partition-id': tenant,
+            'Content-Type': 'application/json',
+            'Authorization': token
+        }
+method = 'POST'
+data = {'email': integration_test_limited_access_user, 'role':'OWNER'}
+f = open(groups_json_path+'/groups_integration_test_limited_access_user_opendes.json')
+groups_data = json.load(f)
+f.close()
 
+for i in groups_data:
+    arr = groups_data[i]
+    for group_name in arr:
+        group_email= group_name+'@'+tenant+'.'+group_domain
+        addMemberUrl=url+'groups/'+group_email+'/members'
+        response = requests.request(method,addMemberUrl, headers=headers, data=json.dumps(data))
+        if response.status_code==200:
+            print('The Entitlements V2 bootstrapping for Integration test limited access user successful for group_email:'+group_email)
+        elif response.status_code==409:
+                print('The Entitlements V2 group for Integration test limited access user already exists for group_email:'+group_email)
+        else:
+            print('The Entitlements V2 bootstrapping for IIntegration test limited access user failed for group_email:'+group_email+'Error response code: '+str(response.status_code))

provider/entitlements-v2-aws/pom.xml

@@ -13,7 +13,7 @@
     <artifactId>entitlements-v2-aws</artifactId>
 
     <properties>
-        <core-lib-aws.version>0.3.16</core-lib-aws.version>
+        <core-lib-aws.version>0.9.1-SNAPSHOT</core-lib-aws.version>
         <reactor.netty.version>0.9.5.RELEASE</reactor.netty.version>
         <reactor.core.version>3.3.0.RELEASE</reactor.core.version>
         <springfox-version>2.7.0</springfox-version>

provider/entitlements-v2-aws/src/main/java/org/opengroup/osdu/entitlements/v2/aws/spi/retrievegroup/AwsRetrieveGroupRepo.java

@@ -40,17 +40,7 @@ import org.springframework.stereotype.Repository;
 import org.springframework.web.context.request.RequestAttributes;
 import org.springframework.web.context.request.RequestContextHolder;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArraySet;
@@ -241,7 +231,9 @@ public class AwsRetrieveGroupRepo implements RetrieveGroupRepo {
     public ParentTreeDto loadAllParents(EntityNode memberNode) {
         Set<String> visited = new CopyOnWriteArraySet<>();
         List<ParentReference> directParents = loadDirectParents(memberNode.getDataPartitionId(), memberNode.getNodeId());
+
         Set<ParentReference> allParents = new HashSet<>(directParents);
+
         visited.add(memberNode.getNodeId());
         Deque<List<ParentReference>> queue = new LinkedList<>();
         if (!directParents.isEmpty()) {
@@ -264,7 +256,16 @@ public class AwsRetrieveGroupRepo implements RetrieveGroupRepo {
                 visited.addAll(allNodeIds);
             });
         }
-        return ParentTreeDto.builder().parentReferences(allParents).maxDepth(maxDepth).build();
+        Set<ParentReference> filteredAllParents = new HashSet<>();
+        Iterator<ParentReference> it = allParents.iterator();
+        while(it.hasNext()){
+            ParentReference pf = it.next();
+          if(pf.getDataPartitionId().equals(memberNode.getDataPartitionId()))
+          {
+              filteredAllParents.add(pf);
+          }
+        }
+        return ParentTreeDto.builder().parentReferences(filteredAllParents).maxDepth(maxDepth).build();
     }
 
     @Override

provider/entitlements-v2-aws/src/main/resources/application.properties

@@ -10,14 +10,9 @@ server.servlet.contextPath=/api/entitlements/v2
 aws.region=${AWS_REGION}
 aws.environment=${ENVIRONMENT}
 app.projectId=evd-ddl-us-services
-app.domain=${SERVICE_DOMAIN_NAME:contoso.com}
+app.domain=${SERVICE_DOMAIN_NAME:example.com}
 
-
-#Need to read TenantInfo table
-
-aws.dynamodb.table.prefix=${ENVIRONMENT}-
-
-aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
+aws.parameter.prefix=/osdu/${ENVIRONMENT}
 
 # Don't know the purpose yet. Needs to be moved to provider side. Issue submitted in GL
 azure.keyvault.url=xx

provider/entitlements-v2-aws/src/main/resources/provisioning/accounts/groups_of_service_principal.json

@@ -83,6 +83,87 @@
     },
     {
       "groupName": "service.workflow.admin"
+    },
+    {
+      "groupName": "service.indexer.admin"
+    },
+    {
+      "groupName": "service.indexer.creator"
+    },
+    {
+      "groupName": "service.indexer.viewer"
+    },
+    {
+      "groupName": "service.unit.admin"
+    },
+    {
+      "groupName": "service.unit.creator"
+    },
+    {
+      "groupName": "service.unit.viewer"
+    },
+    {
+      "groupName": "service.ingest.admin"
+    },
+    {
+      "groupName": "service.ingest.creator"
+    },
+    {
+      "groupName": "service.ingest.viewer"
+    },
+    {
+      "groupName": "service.seismic-store.viewer"
+    },
+    {
+      "groupName": "service.seismic-store.admin"
+    },
+    {
+      "groupName": "service.seismic-store.creator"
+    },
+    {
+      "groupName": "service.binarydms.admin"
+    },
+    {
+      "groupName": "service.binarydms.creator"
+    },
+    {
+      "groupName": "service.binarydms.viewer"
+    },
+    {
+      "groupName": "service.edsdms.admin"
+    },
+    {
+      "groupName": "service.edsdms.creator"
+    },
+    {
+      "groupName": "service.edsdms.viewer"
+    },
+    {
+      "groupName": "service.edsdms.user"
+    },
+    {
+      "groupName": "service.csv-parser.admin"
+    },
+    {
+      "groupName": "service.csv-parser.creator"
+    },
+    {
+      "groupName": "service.csv-parser.viewer"
+    },
+    {
+      "groupName": "service.policy.admin"
+    },
+    {
+      "groupName": "service.policy.creator"
+    },
+    {
+      "groupName": "service.policy.viewer"
+    },
+    {
+      "groupName": "service.delivery.viewer"
+    },
+    {
+      "groupName": "cron.job"
     }
   ]
 }

provider/entitlements-v2-aws/src/main/resources/provisioning/groups/datalake_service_groups.json

@@ -299,6 +299,423 @@
           "name": "users.datalake.admins"
         }
       ]
+    },
+    {
+      "name": "service.indexer.viewer",
+      "description": "The viewer of the indexer service",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.indexer.admin",
+      "description": "Datalake indexer admins",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.indexer.creator",
+      "description": "Datalake indexer creators",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.editors"
+        }
+      ]
+    },
+    {
+      "name": "service.unit.viewer",
+      "description": "The viewer of the unit service",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.unit.admin",
+      "description": "Datalake unit admins",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.unit.creator",
+      "description": "Datalake unit creators",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.editors"
+        }
+      ]
+    },
+    {
+      "name": "service.ingest.viewer",
+      "description": "The viewer of the ingest service",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.ingest.admin",
+      "description": "Datalake ingest admins",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.ingest.creator",
+      "description": "Datalake ingest creators",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.editors"
+        }
+      ]
+    },
+    {
+      "name": "service.seismic-store.viewer",
+      "description": "The viewer of the seismic-store service",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.seismic-store.admin",
+      "description": "Datalake seismic-store admins",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.seismic-store.creator",
+      "description": "Datalake seismic-store creators",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.editors"
+        }
+      ]
+    },
+    {
+      "name": "service.binarydms.viewer",
+      "description": "The viewer of the datalake binarydms service",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.binarydms.admin",
+      "description": "Datalake binarydms admins",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.binarydms.creator",
+      "description": "Datalake binarydms creators",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.editors"
+        }
+      ]