Merge remote-tracking branch 'origin/master' into 67-upgrade-core-common-dependency

Conflicts:
	NOTICE

.gitignore

@@ -7,8 +7,8 @@
 **/dependency-reduced-pom.xml
 **/*.pyc
 /dist/
-.project
-.settings/
-.metadata
-.classpath
-.factorypath
\ No newline at end of file
+**/.prefs
+**/.settings
+**/.factorypath
+**/.classpath
+**/.project

.gitlab-ci.yml

@@ -42,3 +42,8 @@ include:
     
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/ibm.yml"
+
+#Include osdu-gcp-global.yml at k8s common pipeline 
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: 'cloud-providers/osdu-gcp-global.yml'
+

provider/entitlements-v2-aws/maven/settings.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
 -->
 
 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
@@ -63,15 +64,14 @@
         </server>
     </servers>
 
-    <!-- CodeArtifact doesn't support external repos yet that aren't Maven Central.  ETA Q4 2020. -->
-    <!-- <mirrors> -->
-    <!-- <mirror> -->
-    <!-- <id>aws-osdu-dev-maven</id> -->
-    <!-- <name>aws-osdu-dev-maven</name> -->
-    <!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
-    <!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
-    <!-- </mirror> -->
-    <!-- </mirrors> -->
+    <mirrors>
+        <mirror>
+            <id>aws-osdu-dev-maven</id>
+            <name>aws-osdu-dev-maven</name>
+            <url>https://osdu-dev-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url>
+            <mirrorOf>central,!gitlab-os-core-common-maven,!gitlab-os-core-lib-aws-maven</mirrorOf>
+        </mirror>
+    </mirrors>
 
     <activeProfiles>
         <activeProfile>credentialsConfiguration</activeProfile>

provider/entitlements-v2-aws/pom.xml

@@ -13,7 +13,7 @@
     <artifactId>entitlements-v2-aws</artifactId>
 
     <properties>
-        <core-lib-aws.version>0.9.0</core-lib-aws.version>
+        <core-lib-aws.version>0.10.0-SNAPSHOT</core-lib-aws.version>
         <reactor.netty.version>0.9.5.RELEASE</reactor.netty.version>
         <reactor.core.version>3.3.0.RELEASE</reactor.core.version>
         <springfox-version>2.7.0</springfox-version>

provider/entitlements-v2-aws/src/main/java/org/opengroup/osdu/entitlements/v2/aws/AwsAppProperties.java

@@ -34,6 +34,8 @@ public class AwsAppProperties extends AppProperties {
     private String redishost;
     @Value("${redisport}")
     private String redisport;
+    @Value("${rediskey}")
+    private String rediskey;
     @Value("${redis.partition.association}")
     private int redispartitionAssociation;
     @Value("${partition.entitynode}")
@@ -53,6 +55,9 @@ public class AwsAppProperties extends AppProperties {
         return Integer.parseInt(redisport);
     }
 
+    public String getRedisKey() {
+        return rediskey;
+    }
 
     public int getRedisPartitionAssociation() {
         return redispartitionAssociation;

provider/entitlements-v2-aws/src/main/java/org/opengroup/osdu/entitlements/v2/aws/spi/addmember/AwsAddMemberRepo.java

@@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.logging.audit.AuditStatus;
 import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.entitlements.v2.aws.spi.operation.CreateGroupOperationImpl;
 import org.opengroup.osdu.entitlements.v2.di.WhitelistSvcAccBeanConfiguration;
 import org.opengroup.osdu.entitlements.v2.aws.AwsAppProperties;
 import org.opengroup.osdu.entitlements.v2.aws.spi.BaseRepo;
@@ -84,13 +85,23 @@ public class AwsAddMemberRepo extends BaseRepo implements AddMemberRepo {
             throw new AppException(HttpStatus.PRECONDITION_FAILED.value(), HttpStatus.PRECONDITION_FAILED.getReasonPhrase(), String.format("%s's relationship depth quota hit. The relationship depth can't be deeper than %d", addMemberRepoDto.getMemberNode().getNodeId(), MAX_DEPTH));
         }
         final List<String> impactedUsers = childrenUserDto.getChildrenUserIds();
+
         executedCommandsDeque.push(executeParentUpdate(groupEntityNode, addMemberRepoDto.getMemberNode(), addMemberRepoDto.getRole()));
         executedCommandsDeque.push(executeChildrenUpdate(groupEntityNode, addMemberRepoDto.getMemberNode()));
+        Optional<EntityNode> val =retrieveGroupRepo.getEntityNode(addMemberRepoDto.getMemberNode().getNodeId(), addMemberRepoDto.getPartitionId());
+        if (!val.isPresent()){
+            executedCommandsDeque.push(createUserAsAGroupNode(addMemberRepoDto.getMemberNode()));
+        }
         executeUserPartitionAssociationUpdate(groupEntityNode, addMemberRepoDto.getMemberNode(), addMemberRepoDto.getPartitionId())
                 .ifPresent(executedCommandsDeque::push);
+
         return new HashSet<>(impactedUsers);
     }
-
+    private Operation createUserAsAGroupNode(EntityNode memberNode){
+        Operation op = CreateGroupOperationImpl.builder().redisConnector(redisConnector).groupNode(memberNode).log(log).config(config).build();
+        op.execute();
+        return op;
+    }
     private Operation executeParentUpdate(EntityNode groupEntityNode, EntityNode memberNode, Role role) {
         Operation updateParentOperation = AddMemberParentUpdateOperationImpl.builder().redisConnector(redisConnector)
                 .retry(retry).log(log).config(config).groupNode(groupEntityNode).childrenReference(ChildrenReference.createChildrenReference(memberNode, role)).build();

provider/entitlements-v2-aws/src/main/java/org/opengroup/osdu/entitlements/v2/aws/spi/db/RedisConnector.java

@@ -49,23 +49,24 @@ public class RedisConnector {
         getRedisConnectionPool();
     }
 
-    private RedisClient getRedisClient(String host, int port) {
+    private RedisClient getRedisClient(String host, int port, String token) {
         String key = String.format(CONNECTION_KEY_FORMATTER, host, port);
         if (!this.redisClientMap.containsKey(key)) {
-            RedisClient redisClient = RedisClient.create(RedisURI.create(host, port));
+            String temp ="rediss://"+token+"@"+host+":"+port;
+	        RedisClient redisClient = RedisClient.create(RedisURI.create(temp));
             this.redisClientMap.putIfAbsent(key, redisClient);
             return redisClient;
         }
         return this.redisClientMap.get(key);
     }
 
-    private RedisConnectionPool getRedisConnectionPool(String host, int port) {
+    private RedisConnectionPool getRedisConnectionPool(String host, int port, String token) {
         String key = String.format(CONNECTION_KEY_FORMATTER, host, port);
         this.connectionCreationLockMap.putIfAbsent(key, new Object());
         if (!this.redisConnectionPoolMap.containsKey(key)) {
             synchronized (this.connectionCreationLockMap.get(key)) {
                 if (!this.redisConnectionPoolMap.containsKey(key)) {
-                    GenericObjectPool<StatefulRedisConnection<String, String>> pool = ConnectionPoolSupport.createGenericObjectPool(() -> this.getRedisClient(host, port).connect(), this.poolConfig);
+                    GenericObjectPool<StatefulRedisConnection<String, String>> pool = ConnectionPoolSupport.createGenericObjectPool(() -> this.getRedisClient(host, port, token).connect(), this.poolConfig);
                     RedisConnectionPool redisConnectionPool = new RedisConnectionPool(pool);
                     this.redisConnectionPoolMap.putIfAbsent(key, redisConnectionPool);
                 }
@@ -80,13 +81,15 @@ public class RedisConnector {
     public RedisConnectionPool getPartitionRedisConnectionPool(String partitionId) {
          String host = config.getRedisHost();
         int port = config.getRedisPort();
-        return getRedisConnectionPool(host, port);
+        String key = config.getRedisKey();
+        return getRedisConnectionPool(host, port, key);
     }
 
     public RedisConnectionPool getRedisConnectionPool() {
         String host = config.getRedisHost();
         int port = config.getRedisPort();
-        return getRedisConnectionPool(host, port);
+        String key = config.getRedisKey();
+        return getRedisConnectionPool(host, port, key);
     }
 
 }

provider/entitlements-v2-aws/src/main/resources/application.properties

@@ -16,6 +16,7 @@ aws.parameter.prefix=/osdu/${ENVIRONMENT}
 
 redishost=${REDIS_CLUSTER_ENDPOINT}
 redisport=${REDIS_CLUSTER_PORT}
+rediskey=${REDIS_CLUSTER_KEY}
 redis.partition.association=0
 partition.entitynode=1
 partition.parent.ref=2
@@ -30,5 +31,4 @@ server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
 server.ssl.key-password=${SSL_KEY_PASSWORD:}
 server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
 
-
-
+spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
\ No newline at end of file

provider/entitlements-v2-azure/src/main/resources/provisioning/groups/datalake_service_groups.json

@@ -329,6 +329,39 @@
           "name": "users.datalake.admins"
         }
       ]
+    },
+    {
+      "name": "service.dataset.editors",
+      "description": "Editors group for Dataset service",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        }
+      ]
+    },
+    {
+      "name": "service.dataset.viewers",
+      "description": "Viewers group for Dataset service",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.viewers"
+        }
+      ]
     }
   ]
 }

provider/entitlements-v2-azure/src/test/java/org/opengroup/osdu/entitlements/v2/azure/workflow/CreateMembershipsWorkflowSinglePartitionTest.java

@@ -933,6 +933,7 @@ public class CreateMembershipsWorkflowSinglePartitionTest {
                 "service.entitlements.user@common.contoso.com", "service.search.admin@common.contoso.com",
                 "service.storage.admin@common.contoso.com", "users.datalake.viewers@common.contoso.com",
                 "service.storage.creator@common.contoso.com", "service.workflow.admin@common.contoso.com",
-                "data.default.viewers@common.contoso.com"}, performListGroupRequest(servicePrincipal));
+                "data.default.viewers@common.contoso.com", "service.dataset.editors@common.contoso.com",
+                "service.dataset.viewers@common.contoso.com"}, performListGroupRequest(servicePrincipal));
     }
 }

provider/entitlements-v2-ibm/src/main/resources/provisioning/groups/datalake_service_groups.json

@@ -716,6 +716,51 @@
           "name": "users.datalake.editors"
         }
       ]
+    },
+    {
+      "name": "service.legal.viewer",
+      "description": "Legal serice users",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.policy.user",
+      "description": "policy serice users",
+      "members": [
+        {
+          "name": "users.datalake.viewers"
+        },
+        {
+          "name": "users.datalake.editors"
+        },
+        {
+          "name": "users.datalake.admins"
+        },
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
+    },
+    {
+      "name": "service.file.admin",
+      "description": "file serice users",
+      "members": [
+        {
+          "name": "users.datalake.ops"
+        }
+      ]
     }
   ]
 }

testing/entitlements-v2-test-aws/pom.xml

@@ -19,7 +19,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.main.basedir>${project.basedir}</project.main.basedir>
         <jackson.version>2.10.1</jackson.version>
-        <core-lib-aws.version>0.9.0</core-lib-aws.version>
+        <core-lib-aws.version>0.10.0-SNAPSHOT</core-lib-aws.version>
         <junit.version>4.12</junit.version>
         <netty-all.version>4.1.50.Final</netty-all.version>
     </properties>