Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service
1 result
Show changes
Showing
with 1607 additions and 1049 deletions
......@@ -18,7 +18,7 @@ export class IBMSTShelper{
}
public async getCredentials(bucketName: string, keyPath: string,
roleArn: string, flagUpload: boolean, exp: string): Promise<string> {
roleArn: string, roleSessionName: string, flagUpload: boolean, exp: string): Promise<string> {
let policy: any;
if(flagUpload === true)
......@@ -35,7 +35,7 @@ export class IBMSTShelper{
ExternalId: 'OSDUAWS',
Policy: policy,
RoleArn: roleArn,
RoleSessionName: 'OSDUAWSAssumeRoleSession',
RoleSessionName: roleSessionName,
DurationSeconds: expDuration
};
const roleCredentials = await this.sts.assumeRole(stsParams).promise();
......
......@@ -75,7 +75,7 @@ export class Server {
this.app = express();
this.app.use(express.urlencoded({ extended: false }));
this.app.use(express.json());
this.app.use(express.json({ limit: '50MB' }));
this.app.disable('x-powered-by');
this.app.use(cors(this.corsOptions));
this.app.options('*', cors());
......
......@@ -117,29 +117,29 @@ export class SubProjectHandler {
}
Config.enableStrongConsistencyEmulation();
const uuid = uuidv4();
const adminGroup = SubprojectGroups.dataAdminGroup(tenant.name, subproject.name, tenant.esd, uuid);
const viewerGroup = SubprojectGroups.dataViewerGroup(tenant.name, subproject.name, tenant.esd, uuid);
const adminGroupName = adminGroup.split('@')[0];
const viewerGroupName = viewerGroup.split('@')[0];
SubProjectHandler.validateGroupNamesLength(adminGroupName, viewerGroupName, subproject);
// provision new groups
await Promise.all([
AuthGroups.createGroup(userToken, adminGroupName,
// if no admins acl are specified, create a default data group for manage them
if(subproject.acls?.admins?.length === 0) {
const uuid = uuidv4();
const group = SubprojectGroups.dataAdminGroup(tenant.name, subproject.name, tenant.esd, uuid);
const groupName = group.split('@')[0];
SubProjectHandler.validateGroupNameLength(groupName, subproject);
await AuthGroups.createGroup(userToken, groupName,
'seismic dms tenant ' + tenant.name + ' subproject ' + subproject.name + ' admin group',
tenant.esd, req[Config.DE_FORWARD_APPKEY]),
AuthGroups.createGroup(userToken, viewerGroupName,
tenant.esd, req[Config.DE_FORWARD_APPKEY]);
subproject.acls.admins = [ group ];
}
// if no viewers acls are specified, create a default data group for manage them
if(subproject.acls?.viewers?.length === 0 ) {
const uuid = uuidv4();
const group = SubprojectGroups.dataViewerGroup(tenant.name, subproject.name, tenant.esd, uuid);
const groupName = group.split('@')[0];
SubProjectHandler.validateGroupNameLength(groupName, subproject);
await AuthGroups.createGroup(userToken, groupName,
'seismic dms tenant ' + tenant.name + ' subproject ' + subproject.name + ' viewer group',
tenant.esd, req[Config.DE_FORWARD_APPKEY])]
);
tenant.esd, req[Config.DE_FORWARD_APPKEY]);
subproject.acls.viewers = [ group ];
subproject.acls.admins = subproject.acls.admins ? subproject.acls.admins.concat([adminGroup])
.filter((group, index, self) => self.indexOf(group) === index) : [adminGroup];
subproject.acls.viewers = subproject.acls.viewers ? subproject.acls.viewers.concat([viewerGroup])
.filter((group, index, self) => self.indexOf(group) === index) : [viewerGroup];
}
// Create the storage resource
subproject.gcs_bucket = await this.getBucketName(tenant);
......@@ -152,9 +152,9 @@ export class SubProjectHandler {
if (req.body?.admin && req.body?.admin !== Utils.getPropertyFromTokenPayload(
userToken, Config.USER_ID_CLAIM_FOR_ENTITLEMENTS_SVC)) {
await Promise.all([
AuthGroups.addUserToGroup(userToken, adminGroup, req.body.admin,
AuthGroups.addUserToGroup(userToken, subproject.acls.admins[0], req.body.admin,
tenant.esd, req[Config.DE_FORWARD_APPKEY], UserRoles.Owner, true),
AuthGroups.addUserToGroup(userToken, viewerGroup, req.body.admin,
AuthGroups.addUserToGroup(userToken, subproject.acls.viewers[0], req.body.admin,
tenant.esd, req[Config.DE_FORWARD_APPKEY], UserRoles.Owner, true)
]);
......@@ -236,8 +236,12 @@ export class SubProjectHandler {
const viewerSubprojectDataGroups = subproject.acls.viewers.filter(group => group.match(dataGroupRegex));
const subprojectDataGroups = adminSubprojectDataGroups.concat(viewerSubprojectDataGroups);
for (const group of subprojectDataGroups) {
await AuthGroups.deleteGroup(
req.headers.authorization, group, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Utils.exponentialBackOff(AuthGroups.deleteGroup(
req.headers.authorization, group, tenant.esd,
req[Config.DE_FORWARD_APPKEY])).catch((error)=>{
console.error(error);
});
}
......@@ -270,25 +274,24 @@ export class SubProjectHandler {
await Auth.isUserAuthorized(req.headers.authorization,
SubprojectAuth.getAuthGroups(subproject, AuthRoles.admin), tenant.esd, req[Config.DE_FORWARD_APPKEY]);
if (parsedUserInput.acls) {
subproject.acls = parsedUserInput.acls;
}
// Updated the access policy
if (parsedUserInput.access_policy) {
this.validateAccessPolicy(parsedUserInput.access_policy, subproject.access_policy);
subproject.access_policy = parsedUserInput.access_policy;
}
const adminGroups = [SubprojectGroups.serviceAdminGroup(tenant.name, subproject.name, tenant.esd),
SubprojectGroups.serviceEditorGroup(tenant.name, subproject.name, tenant.esd)];
const viewerGroups = [SubprojectGroups.serviceViewerGroup(tenant.name, subproject.name, tenant.esd)];
subproject.acls.admins = subproject.acls.admins ? subproject.acls.admins
.filter((group, index, self) => self.indexOf(group) === index) : adminGroups;
subproject.acls.viewers = subproject.acls.viewers ? subproject.acls.viewers
.filter((group, index, self) => self.indexOf(group) === index) : viewerGroups;
// save and replace default SDMS ACLs to delete at the end
const dataGroupsToDelete = [] as string[];
if (parsedUserInput?.acls?.admins?.length > 0) {
const defaultDataGroupRegEx = SubprojectGroups.dataGroupNameRegExp(tenant.name, subproject.name);
dataGroupsToDelete.concat(subproject.acls.admins.filter((group) => group.match(defaultDataGroupRegEx)));
subproject.acls.admins = parsedUserInput.acls.admins;
}
if (parsedUserInput?.acls?.viewers?.length > 0) {
const defaultDataGroupRegEx = SubprojectGroups.dataGroupNameRegExp(tenant.name, subproject.name);
dataGroupsToDelete.concat(subproject.acls.viewers.filter((group) => group.match(defaultDataGroupRegEx)));
subproject.acls.viewers = parsedUserInput.acls.viewers;
}
// update the legal tag (check if the new one is valid)
if (parsedUserInput.ltag) {
......@@ -326,8 +329,15 @@ export class SubProjectHandler {
// Update the subproject metadata
await SubProjectDAO.register(journalClient, subproject);
return subproject;
// delete replaced default data group ACLs (detached retry ops)
for (const group of dataGroupsToDelete) {
Utils.exponentialBackOff(AuthGroups.deleteGroup(
req.headers.authorization, group, tenant.esd, req[Config.DE_FORWARD_APPKEY])).catch((error) => {
console.error(error);
});
}
return subproject;
}
// List the subprojects in a tenant
......@@ -382,12 +392,8 @@ export class SubProjectHandler {
}
// Ensure the group name respect CSP imposed length limits
private static validateGroupNamesLength(adminGroupName: string, viewerGroupName: string,
subproject: SubProjectModel): boolean {
const allowedSubprojectLen = Config.DES_GROUP_CHAR_LIMIT - Math.max(
adminGroupName.length, viewerGroupName.length
);
private static validateGroupNameLength(groupName: string, subproject: SubProjectModel) {
const allowedSubprojectLen = Config.DES_GROUP_CHAR_LIMIT - groupName.length;
if (allowedSubprojectLen < 0) {
const maxChar = subproject.name.length - Math.abs(allowedSubprojectLen);
......@@ -396,9 +402,6 @@ export class SubProjectHandler {
subproject.name.length + ' characters), for the ' + subproject.tenant + ' tenant. ' +
'The subproject name must not be longer than ' + maxChar + ' characters'));
}
return true;
}
// Ensure the access policy is not changed if already set as 'dataset'
......@@ -409,6 +412,4 @@ export class SubProjectHandler {
}
}
}
......@@ -31,10 +31,17 @@ export class SubProjectParser {
subproject.tenant = req.params.tenantid;
subproject.ltag = req.headers.ltag as string;
// optional parameters
subproject.acls = (req.body && req.body.acls) ? req.body.acls : { 'admins': [], 'viewers': [] };
subproject.access_policy = (req.body && req.body.access_policy) ?
req.body.access_policy : Config.UNIFORM_ACCESS_POLICY;
// If not specified, set the acl as empty array. A default acl group will be later created for these.
subproject.acls = req.body?.acls || { 'admins': [], 'viewers': [] };
if(req.body?.acl) {
const aclKeys = Object.keys(req.body.acls);
subproject.acls['admins'] = ('admins' in aclKeys) ? subproject.acls['admins'].sort() : [];
subproject.acls['viewers'] = ('viewers' in aclKeys) ? subproject.acls['viewers'].sort() : [];
}
// set the dataset level access acl (uniform by default)
subproject.access_policy = req.body?.access_policy || Config.UNIFORM_ACCESS_POLICY;
// check user input params
Params.checkString(subproject.admin, 'admin', false);
......@@ -99,4 +106,4 @@ export class SubProjectParser {
}
}
}
\ No newline at end of file
}
......@@ -188,13 +188,13 @@ export class UserHandler {
const dataGroupRegex = SubprojectGroups.dataGroupNameRegExp(tenant.name, subproject.name);
const adminSubprojectDataGroups = subproject.acls.admins.filter((group) => group.match(dataGroupRegex));
const viewerSuprojectDataGroups = subproject.acls.viewers.filter(group => group.match(dataGroupRegex));
const adminGroups = subprojectAdminServiceGroups.concat(adminSubprojectDataGroups);
const viewerGroups = subprojectViewerServiceGroups.concat(viewerSuprojectDataGroups);
const viewerSubprojectDataGroups = subproject.acls.viewers.filter(group => group.match(dataGroupRegex));
let adminGroups = subprojectAdminServiceGroups.concat(adminSubprojectDataGroups);
let viewerGroups = subprojectViewerServiceGroups.concat(viewerSubprojectDataGroups);
adminGroups= adminGroups.length ? adminGroups : [subproject.acls.admins.sort()[0]];
viewerGroups = viewerGroups .length ? viewerGroups : [subproject.acls.viewers.sort()[0]];
if (userGroupRole === AuthRoles.admin || userGroupRole === AuthRoles.editor) {
await this.addUserAsAdmin(adminGroups, viewerGroups, tenant, req, userEmail);
......@@ -250,17 +250,46 @@ export class UserHandler {
const result = await UserHandler.listUsersInAuthGroups(datasetOUT.acls.admins, datasetOUT.acls.viewers,
req, tenant);
await UserHandler.findAndRemoveUser(result, userEmail, datasetOUT, tenant, req);
await UserHandler.findAndRemoveUserFromDataset(result, userEmail, datasetOUT, tenant, req);
}
} else if (sdPath.subproject) {
const result = await UserHandler.listUsersInAuthGroups(subproject.acls.admins, subproject.acls.viewers,
req, tenant);
const serviceGroupRegex = SubprojectGroups.serviceGroupNameRegExp(tenant.name, subproject.name);
const subprojectAdminServiceGroups = subproject.acls.admins
.filter((group) => group.match(serviceGroupRegex));
const subprojectViewerServiceGroups = subproject.acls.viewers
.filter((group) => group.match(serviceGroupRegex));
const dataGroupRegex = SubprojectGroups.dataGroupNameRegExp(tenant.name, subproject.name);
const adminSubprojectDataGroups = subproject.acls.admins.filter((group) => group.match(dataGroupRegex));
const viewerSubprojectDataGroups = subproject.acls.viewers.filter(group => group.match(dataGroupRegex));
const adminGroups = subprojectAdminServiceGroups.concat(adminSubprojectDataGroups);
const viewerGroups = subprojectViewerServiceGroups.concat(viewerSubprojectDataGroups);
await UserHandler.findAndRemoveUser(result, userEmail, subproject, tenant, req);
let userGroups = [];
if(adminGroups.length === 0) {
userGroups = await AuthGroups.getUserGroups(req.headers.authorization,
tenant.esd, req[Config.DE_FORWARD_APPKEY]);
adminGroups.push(userGroups.filter(group => subproject.acls.admins.includes(group.email))
.map(group => group.email).sort()[0]);
}
if(viewerGroups.length === 0) {
if (userGroups.length === 0) {
userGroups = await AuthGroups.getUserGroups(req.headers.authorization,
tenant.esd, req[Config.DE_FORWARD_APPKEY]);
}
viewerGroups.push(userGroups.filter(group => subproject.acls.viewers.includes(group.email))
.map(group => group.email).sort()[0]);
}
await UserHandler.removeUserFromAuthGroups(adminGroups, viewerGroups, tenant, req, userEmail);
} else {
throw (Error.make(Error.Status.BAD_REQUEST,
'Please use Delfi portal to remove users from ' + tenant.name + ' tenant'));
......@@ -268,8 +297,8 @@ export class UserHandler {
}
private static async findAndRemoveUser(userListInAuthGroups: any[], userEmail: string,
datastoreEntity: DatasetModel | SubProjectModel, tenant: TenantModel, req) {
private static async findAndRemoveUserFromDataset(userListInAuthGroups: any[], userEmail: string,
datastoreEntity: DatasetModel, tenant: TenantModel, req) {
const admins = new Set();
const viewers = new Set();
......@@ -280,9 +309,7 @@ export class UserHandler {
} else {
viewers.add(lst[0]);
}
}
});
if (admins.has(userEmail)) {
......@@ -359,14 +386,12 @@ export class UserHandler {
let users = [];
for (const adminGroup of admins) {
const result = (await AuthGroups.listUsersInGroup(req.headers.authorization, adminGroup, tenant.esd,
req[Config.DE_FORWARD_APPKEY]));
users = users.concat(result.map((el) => [el.email, 'admin']));
}
for (const viewerGroup of viewers) {
const result = (await AuthGroups.listUsersInGroup(req.headers.authorization, viewerGroup, tenant.esd,
req[Config.DE_FORWARD_APPKEY]));
......
......@@ -136,4 +136,24 @@ export class Utils {
}
// retry on error using exponential retry backOff strategy.
// wait*2^1+e, wait*2^2+e, wait*2^3+e, wait*2^(retryMaxAttempts)+e
// 200ms 400ms 800ms 1600ms ...
public static async exponentialBackOff(
methodToCall: any, retryMaxAttempts = 5): Promise<void> {
let retries = 0;
const waitTime = 200;
while (true) {
try {
return await methodToCall();
} catch (error) {
if (retryMaxAttempts === ++retries) {
throw (error);
}
await new Promise(resolve => setTimeout(
resolve, ((2 ** (retries - 1)) * waitTime) + Math.random() * 100));
}
}
}
}
......@@ -15,7 +15,8 @@ Postman collection with API requests that check the basic functionality of the s
| legaltag01 | valid legal tag registered in Legal Service | Required |
| legaltag02 | another valid legal tag registered in Legal Service | Required |
| newuser | valid user registered in data partition other then STOKEN issued for | Required if _VCS_Provider_ is not set |
| VCS_Provider | possible values are ```true``` for script or ```gitlab``` for Newman. Need to skip USER and IMPTOKEN API endpoints test | Required if _newuser_ is not set |
| newusergroup | valid user group registered in data partition | Required if _VCS_Provider_ is not set |
| VCS_Provider | possible values are ```true``` for script or ```gitlab``` for Newman. It is needed to skip USER and IMPTOKEN API endpoints test | Required if _newuser_ and/or _newusergroup_ is not set |
| SVC_API_KEY | historical variables and could be any string | Optional |
| DE_APP_KEY | historical variables and could be any string | Optional |
---
......
{
"id": "79282296-739d-4c80-a65c-86e676354e5e",
"id": "8f901ba8-9441-41ea-a086-e0a04a716b3e",
"name": "seistore-test-e2e-gitlab-dafaults",
"values": [
{
......@@ -47,6 +47,12 @@
"value": "#{NEWUSEREMAIL}#",
"enabled": true
},
{
"key": "newuser_group",
"value": "#{NEWUSERGROUP}#",
"type": "default",
"enabled": true
},
{
"key": "DE_APP_KEY",
"value": "#{DE_APP_KEY}#",
......@@ -64,6 +70,6 @@
}
],
"_postman_variable_scope": "environment",
"_postman_exported_at": "2021-12-03T15:32:49.050Z",
"_postman_exported_using": "Postman/9.3.0"
"_postman_exported_at": "2022-09-21T08:35:08.061Z",
"_postman_exported_using": "Postman/9.31.9"
}
\ No newline at end of file
......@@ -20,8 +20,8 @@ usage() {
printf "\n[USAGE] ./e2e/tests/run_e2e_tests.sh --seistore-svc-url=... " \
"--seistore-svc-api-key=... --user-idtoken=... --tenant=..." \
"--datapartition=... --legaltag01=... --legaltag02=... " \
"--newuser(optional)=... --VCS-provider(optional)=... --de-app-key(optional)=... " \
"--admin-email(optional)=... --subproject(optional)=... \n "
"--newuser(optional)=... --newusergroup(optional)=... --VCS-provider(optional)=... " \
"--admin-email(optional)=... --de-app-key(optional)=... --subproject(optional)=... \n "
printf "\n[ERROR] %s\n" "$1"
}
......@@ -39,7 +39,8 @@ usage() {
# argument [datapartition] data partition id - required
# argument [legaltag01] test legal tag - required
# argument [legaltag02] test legal tag - required
# argument [newuser] user email for a new user to add partition id - required if [VCS-Provider] is not 'gitlab'
# argument [newuser] user email for a new user to be added into subproject - required if [VCS-Provider] is not 'gitlab'
# argument [newusergroup] user group email for a group to be added into subproject - required if [VCS-Provider] is not 'gitlab'
# argument [VCS-Provider] valid value is 'gitlab'. Provided will skip USER and IMPTOKEN API endpoints tests - optional
# argument [de-app-key] DELFI application key - optional
# argument [admin-email] user credentail email - optional (deprecated)
......@@ -79,6 +80,10 @@ case $i in
newuser="${i#*=}"
shift
;;
--newusergroup=*)
newusergroup="${i#*=}"
shift
;;
--de-app-key=*)
de_app_key="${i#*=}"
shift
......@@ -115,6 +120,7 @@ if [[ "${VCS_Provider}" == true || "${VCS_Provider}" == "true" || "${VCS_Provide
VCS_Provider="gitlab"
else
if [ -z "${newuser}" ]; then usage "newuser not defined" && exit 1; fi
if [ -z "${newusergroup}" ]; then usage "newusergroup not defined" && exit 1; fi
VCS_Provider="any"
fi
......@@ -143,6 +149,7 @@ printf "%s\n" "datapartition = ${datapartition}"
printf "%s\n" "legaltag01 = ${legaltag01}"
printf "%s\n" "legaltag02 = ${legaltag02}"
printf "%s\n" "newuser = ${newuser}"
printf "%s\n" "newusergroup = ${newusergroup}"
printf "%s\n" "VCS_Provider = ${VCS_Provider}"
printf "%s\n" "subproject = ${subproject}"
printf "%s\n" "--------------------------------------------"
......@@ -158,6 +165,7 @@ sed -i "s/#{DATAPARTITION}#/${datapartition}/g" ./tests/e2e/postman_env.json
sed -i "s/#{LEGALTAG01}#/${legaltag01}/g" ./tests/e2e/postman_env.json
sed -i "s/#{LEGALTAG02}#/${legaltag02}/g" ./tests/e2e/postman_env.json
sed -i "s/#{NEWUSEREMAIL}#/${newuser}/g" ./tests/e2e/postman_env.json
sed -i "s/#{NEWUSERGROUP}#/${newusergroup}/g" ./tests/e2e/postman_env.json
sed -i "s/#{VCS_PROVIDER}#/${VCS_Provider}/g" ./tests/e2e/postman_env.json
sed -i "s/#{DE_APP_KEY}#/${de_app_key}/g" ./tests/e2e/postman_env.json
sed -i "s/#{SUBPROJECT}#/${subproject}/g" ./tests/e2e/postman_env.json
......
......@@ -138,6 +138,11 @@ export class TestAuth {
} catch (e) { Tx.check403(e.error.code, done); }
});
Tx.test(async (done: any) => {
this.sandbox.stub(ImpTokenDAO, 'getImpTokenBody').returns({ resources: [] } as ImpTokenBodyModel);
Tx.checkFalse(await Auth.isReadAuthorized(this.impToken, [], tenant, 's', 'appkey', undefined, false), done);
});
}
private static usersGroup() {
......
......@@ -49,7 +49,7 @@ export class TestCredentials {
Tx.sectionInit('service account email');
Tx.testExp(async (done: any) => {
this.sandbox.stub(axios, 'get').resolves();
this.sandbox.stub(axios, 'get').resolves({data: 'email'});
await this.credentials.getServiceAccountEmail();
done();
});
......
This diff is collapsed.
......@@ -58,7 +58,7 @@ azure_swagger_test:
echo 'empty'
only:
- $DISABLED == 'true'
azure-publish:
needs: []
script:
......
......@@ -84,7 +84,7 @@
- >
if [[ "$CI_COMMIT_TAG" != "" ]];
then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_TAG";
elif [[ "$CI_COMMIT_REF_NAME" = "master" ]];
elif [[ "$CI_COMMIT_REF_NAME" = "$CI_DEFAULT_BRANCH" ]];
then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest"; fi
- docker build -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file $SEISMIC_OSDU_GCP_DOCKERFILE_NAME .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
......