handler.ts 18.2 KB
Newer Older
Diego Molteni's avatar
Diego Molteni committed
1
// ============================================================================
2
// Copyright 2017-2021, Schlumberger
Diego Molteni's avatar
Diego Molteni committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ============================================================================

import { Request as expRequest, Response as expResponse } from 'express';
18
import { v4 as uuidv4 } from 'uuid';
Diego Molteni's avatar
Diego Molteni committed
19
20
import { SubProjectModel } from '.';
import { Auth, AuthGroups } from '../../auth';
21
import { Config, JournalFactoryTenantClient, LoggerFactory, StorageFactory } from '../../cloud';
22
import { SeistoreFactory } from '../../cloud/seistore';
23
import { Error, Feature, FeatureFlags, Response } from '../../shared';
Diego Molteni's avatar
Diego Molteni committed
24
25
26
27
28
29
30
31
import { DatasetDAO, PaginationModel } from '../dataset';
import { TenantGroups, TenantModel } from '../tenant';
import { TenantDAO } from '../tenant/dao';
import { SubProjectDAO } from './dao';
import { SubprojectGroups } from './groups';
import { SubProjectOP } from './optype';
import { SubProjectParser } from './parser';

32

Diego Molteni's avatar
Diego Molteni committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
export class SubProjectHandler {

    // handler for the [ /subproject ] endpoints
    public static async handler(req: expRequest, res: expResponse, op: SubProjectOP) {

        try {

            if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
                // subproject endpoints are not available with impersonation token
                if (Auth.isImpersonationToken(req.headers.authorization)) {
                    throw (Error.make(Error.Status.PERMISSION_DENIED,
                        'subproject endpoints not available' +
                        ' with an impersonation token as Auth credentials.'));
                }
            }

            const tenant = await TenantDAO.get(req.params.tenantid);

            if (op === SubProjectOP.Create) {

                const subproject = await this.create(req, tenant);
                delete (subproject as any).service_account; // we don't want to return it
                Response.writeOK(res, subproject);

            } else if (op === SubProjectOP.Get) {

                const subproject = await this.get(req, tenant);
                delete (subproject as any).service_account; // we don't want to return it
                Response.writeOK(res, subproject);

            } else if (op === SubProjectOP.Delete) {

                await this.delete(req, tenant);
                Response.writeOK(res);

            } else if (op === SubProjectOP.Patch) {

                const subproject = await this.patch(req, tenant);
                delete (subproject as any).service_account; // we don't want to return it
                Response.writeOK(res, subproject);

            } else if (op === SubProjectOP.List) {

                const subprojects = await this.list(req, tenant);
                for (const item of subprojects) { delete (item as any).service_account; } // we don't want to return it
                Response.writeOK(res, subprojects);

            } else { throw (Error.make(Error.Status.UNKNOWN, 'Internal Server Error')); }

        } catch (error) { Response.writeError(res, error); }

    }

    // create a new subproject
    private static async create(req: expRequest, tenant: TenantModel): Promise<SubProjectModel> {

        // Parse input parameters
90
        const subproject = await SubProjectParser.create(req);
Diego Molteni's avatar
Diego Molteni committed
91
        const userToken = req.headers.authorization;
92
93
        const userEmail = await SeistoreFactory.build(
            Config.CLOUDPROVIDER).getEmailFromTokenPayload(req.headers.authorization, true);
Diego Molteni's avatar
Diego Molteni committed
94

95
96
        subproject.admin = subproject.admin || userEmail;

97
98
99
100
101
102
        // enforce the datasets schema by key for newly create subproject.
        // this will mainly affect google for which the initial implementation
        // of the journal was query-based (lack in performance)
        // other cloud providers already implement ref by key.
        subproject.enforce_key = Config.ENFORCE_SCHEMA_BY_KEY;

Diego Molteni's avatar
Diego Molteni committed
103
104
105
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            // Check if user is a tenant admin
            await Auth.isUserAuthorized(
Diego Molteni's avatar
Diego Molteni committed
106
                userToken, [TenantGroups.adminGroup(tenant)], tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
107
        }
Diego Molteni's avatar
Diego Molteni committed
108
        if (FeatureFlags.isEnabled(Feature.LEGALTAG) && subproject.ltag) {
Diego Molteni's avatar
Diego Molteni committed
109
110
111
112
113
114
115
116
117
            // Check if the legal tag is valid
            await Auth.isLegalTagValid(req.headers.authorization,
                subproject.ltag, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // Check if the subproject already exists
118
        if (await SubProjectDAO.exist(journalClient, subproject.tenant, subproject.name)) {
Diego Molteni's avatar
Diego Molteni committed
119
120
121
122
123
            throw (Error.make(Error.Status.ALREADY_EXISTS,
                'The subproject ' + subproject.name +
                ' already exists in the tenant project ' + subproject.tenant));
        }

124
125
126
        const uuid = uuidv4();
        const adminGroup = SubprojectGroups.dataAdminGroup(tenant.name, subproject.name, tenant.esd, uuid);
        const viewerGroup = SubprojectGroups.dataViewerGroup(tenant.name, subproject.name, tenant.esd, uuid);
127

128
129
        const adminGroupName = adminGroup.split('@')[0];
        const viewerGroupName = viewerGroup.split('@')[0];
130

131
        SubProjectHandler.validateGroupNamesLength(adminGroupName, viewerGroupName, subproject);
132

Diego Molteni's avatar
Diego Molteni committed
133
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
134
135
136
137
138
139
140
            // provision new groups
            await AuthGroups.createGroup(userToken, adminGroupName,
                'seismic dms tenant ' + tenant.name + ' subproject ' + subproject.name + ' admin group',
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
            await AuthGroups.createGroup(userToken, viewerGroupName,
                'seismic dms tenant ' + tenant.name + ' subproject ' + subproject.name + ' editor group',
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
141
142
143
144
        }

        subproject.gcs_bucket = await this.getBucketName(tenant);

145
        subproject.acls.admins = subproject.acls.admins ? subproject.acls.admins.concat([adminGroup])
146
            .filter((group, index, self) => self.indexOf(group) === index) : [adminGroup];
147
        subproject.acls.viewers = subproject.acls.viewers ? subproject.acls.viewers.concat([viewerGroup])
148
            .filter((group, index, self) => self.indexOf(group) === index) : [viewerGroup];
Diego Molteni's avatar
Diego Molteni committed
149

Diego Molteni's avatar
Diego Molteni committed
150
151
152
153
        // Create the GCS bucket resource
        const storage = StorageFactory.build(Config.CLOUDPROVIDER, tenant);
        await storage.createBucket(
            subproject.gcs_bucket,
Diego Molteni's avatar
Diego Molteni committed
154
            subproject.storage_location, subproject.storage_class);
Diego Molteni's avatar
Diego Molteni committed
155

Diego Molteni's avatar
Diego Molteni committed
156

Diego Molteni's avatar
Diego Molteni committed
157
        // Register the subproject
158
        await SubProjectDAO.register(journalClient, subproject);
Diego Molteni's avatar
Diego Molteni committed
159
160
161
162

        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            // if admin is not the requestor, assign the admin and rm the requestor, has to be a sequential op
            if (subproject.admin !== userEmail) {
163
164

                await AuthGroups.addUserToGroup(userToken, adminGroup, subproject.admin,
Diego Molteni's avatar
Diego Molteni committed
165
                    tenant.esd, req[Config.DE_FORWARD_APPKEY], 'OWNER', true);
166
167

                await AuthGroups.addUserToGroup(userToken, viewerGroup, subproject.admin,
Diego Molteni's avatar
Diego Molteni committed
168
                    tenant.esd, req[Config.DE_FORWARD_APPKEY], 'OWNER', true);
169

Diego Molteni's avatar
Diego Molteni committed
170
171
172
            }
        }

173
        const status = await SeistoreFactory.build(Config.CLOUDPROVIDER).notifySubprojectCreationStatus(subproject, 'created');
174
175
176
177
178
179

        if (!status) {
            LoggerFactory.build(Config.CLOUDPROVIDER)
                .error('Unable to publish creation status for subproject ' + subproject.name);
        }

Diego Molteni's avatar
Diego Molteni committed
180
181
182
183
184
185
186
187
188
189
        return subproject;
    }

    // retrieve the subproject metadata
    private static async get(req: expRequest, tenant: TenantModel): Promise<SubProjectModel> {

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // get subproject
190
        const subproject = await SubProjectDAO.get(journalClient, tenant.name, req.params.subprojectid);
Diego Molteni's avatar
Diego Molteni committed
191

Diego Molteni's avatar
Diego Molteni committed
192
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
193
            // Check if user is member of any of the subproject acl admin groups
Diego Molteni's avatar
Diego Molteni committed
194
195
196
197
198
            await Auth.isUserAuthorized(req.headers.authorization,
                subproject.acls.admins, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }


Diego Molteni's avatar
Diego Molteni committed
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
        if (FeatureFlags.isEnabled(Feature.LEGALTAG)) {
            // Check if the legal tag is valid
            if (subproject.ltag) {
                // [TODO] we should always have ltag. some subprojects does not have it (the old ones)
                await Auth.isLegalTagValid(req.headers.authorization,
                    subproject.ltag, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
            }
        }

        return subproject;

    }

    // delete the subproject
    private static async delete(req: expRequest, tenant: TenantModel) {

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // get the subproject metadata
219
        const subproject = await SubProjectDAO.get(journalClient, tenant.name, req.params.subprojectid);
Diego Molteni's avatar
Diego Molteni committed
220

Daniel Perez's avatar
Daniel Perez committed
221
        // Only tenant admins are allowed to delete the subproject
Diego Molteni's avatar
Diego Molteni committed
222
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
223
224
            await Auth.isUserAuthorized(
                req.headers.authorization, [AuthGroups.datalakeUserAdminGroupEmail(tenant.esd)],
Diego Molteni's avatar
Diego Molteni committed
225
226
227
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }

Diego Molteni's avatar
Diego Molteni committed
228
229
230
231
        const storage = StorageFactory.build(Config.CLOUDPROVIDER, tenant);

        await Promise.all([
            // delete the subproject metadata from Datastore
232
            SubProjectDAO.delete(journalClient, tenant.name, subproject.name),
Diego Molteni's avatar
Diego Molteni committed
233
234
235
236
237
238
            // delete all datasets metadata from Datastore.
            DatasetDAO.deleteAll(journalClient, tenant.name, subproject.name),
            // delete the subproject associated bucket. This operation will delete all subproject data in GCS.
            storage.deleteFiles(subproject.gcs_bucket),
        ]);

Diego Molteni's avatar
Diego Molteni committed
239
        const serviceGroupRegex = SubprojectGroups.serviceGroupNameRegExp(tenant.name, subproject.name);
240
        const subprojectServiceGroups = subproject.acls.admins.filter((group) => group.match(serviceGroupRegex));
Diego Molteni's avatar
Diego Molteni committed
241
242

        const dataGroupRegex = SubprojectGroups.dataGroupNameRegExp(tenant.name, subproject.name);
243
        const adminSubprojectDataGroups = subproject.acls.admins.filter((group) => group.match(dataGroupRegex));
244
245
        const viewerSubprojectDataGroups = subproject.acls.viewers.filter(group => group.match(dataGroupRegex));
        const subprojectDataGroups = adminSubprojectDataGroups.concat(viewerSubprojectDataGroups);
Diego Molteni's avatar
Diego Molteni committed
246

Diego Molteni's avatar
Diego Molteni committed
247
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
248
            for (const group of subprojectDataGroups) {
249
                await AuthGroups.deleteGroup(
Diego Molteni's avatar
Diego Molteni committed
250
251
                    req.headers.authorization, group, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
            }
Diego Molteni's avatar
Diego Molteni committed
252
253
254
        }

        // delete the bucket resource (to perform after files deletions)
255
        // tslint:disable-next-line: no-floating-promises no-console (we want it async)
256
257
258
259
        storage.deleteBucket(subproject.gcs_bucket).catch((error) => {
            LoggerFactory.build(Config.CLOUDPROVIDER).error(JSON.stringify(error));
        });

Diego Molteni's avatar
Diego Molteni committed
260
261
262
263
264
265
266
    }

    // delete the subproject
    private static async patch(req: expRequest, tenant: TenantModel) {

        const parsedUserInput = SubProjectParser.patch(req);

267
        // bad request if there are no field to patch
Diego Molteni's avatar
Diego Molteni committed
268
269
270
271
272
273
274
        if (!parsedUserInput.ltag) {
            throw (Error.make(Error.Status.BAD_REQUEST,
                'The request does not contain any field to patch'));
        }

        // init journalClient client and key
        const journalClient = JournalFactoryTenantClient.get(tenant);
Diego Molteni's avatar
Diego Molteni committed
275

Diego Molteni's avatar
Diego Molteni committed
276
        // get subproject
277
        const subproject = await SubProjectDAO.get(journalClient, tenant.name, req.params.subprojectid);
Diego Molteni's avatar
Diego Molteni committed
278

Diego Molteni's avatar
Diego Molteni committed
279
280
281
282
283
284
285
286

        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            // Check if user is a subproject admin
            await Auth.isUserAuthorized(req.headers.authorization,
                subproject.acls.admins, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }

        if (parsedUserInput.acls) {
287
            subproject.acls = parsedUserInput.acls;
Diego Molteni's avatar
Diego Molteni committed
288
289
        }

290
291
292
293
        if (parsedUserInput.access_policy) {
            this.validateAccessPolicy(parsedUserInput.access_policy, subproject.access_policy);
            subproject.access_policy = parsedUserInput.access_policy;
        }
Diego Molteni's avatar
Diego Molteni committed
294

295
        const adminGroups = [SubprojectGroups.serviceAdminGroup(tenant.name, subproject.name, tenant.esd),
296
297
        SubprojectGroups.serviceEditorGroup(tenant.name, subproject.name, tenant.esd)];
        const viewerGroups = [SubprojectGroups.serviceViewerGroup(tenant.name, subproject.name, tenant.esd)];
Diego Molteni's avatar
Diego Molteni committed
298
299

        subproject.acls.admins = subproject.acls.admins ? subproject.acls.admins
300
            .filter((group, index, self) => self.indexOf(group) === index) : adminGroups;
Diego Molteni's avatar
Diego Molteni committed
301
        subproject.acls.viewers = subproject.acls.viewers ? subproject.acls.viewers
302
            .filter((group, index, self) => self.indexOf(group) === index) : viewerGroups;
Diego Molteni's avatar
Diego Molteni committed
303
304


Diego Molteni's avatar
Diego Molteni committed
305
306
307
        // update the legal tag (check if the new one is valid)
        if (parsedUserInput.ltag) {

Diego Molteni's avatar
Diego Molteni committed
308
            if (FeatureFlags.isEnabled(Feature.LEGALTAG)) {
Diego Molteni's avatar
Diego Molteni committed
309
310
311
312
                await Auth.isLegalTagValid(
                    req.headers.authorization, parsedUserInput.ltag, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
            }

313
314
            const originalSubprojectLtag = subproject.ltag;
            if (originalSubprojectLtag !== parsedUserInput.ltag) {
Diego Molteni's avatar
Diego Molteni committed
315
316
317
318
319
320
321
322
323
324

                // update the subproject ltag
                subproject.ltag = parsedUserInput.ltag;
                // recursively update all datasets legal tag
                if (parsedUserInput.recursive) {
                    const pagination = { limit: 1000, cursor: undefined } as PaginationModel;
                    do {
                        const output = await DatasetDAO.listDatasets(
                            journalClient, subproject.tenant, subproject.name, pagination);
                        const datasets = output.datasets.filter(dataset => {
325
                            return dataset.data.ltag === originalSubprojectLtag;
Diego Molteni's avatar
Diego Molteni committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
                        });
                        for (const dataset of datasets) {
                            dataset.data.ltag = parsedUserInput.ltag;
                        }
                        if (datasets.length > 0) {
                            await DatasetDAO.updateAll(journalClient, datasets);
                        }
                        pagination.cursor = output.nextPageCursor;
                    } while (pagination.cursor);
                }

            }

        }

341
342
        // Update the subproject metadata
        await SubProjectDAO.register(journalClient, subproject);
Diego Molteni's avatar
Diego Molteni committed
343

344
        return subproject;
Diego Molteni's avatar
Diego Molteni committed
345
346
347
348
349
350
351
352
353

    }

    // list the subprojects in a tenant
    private static async list(req: expRequest, tenant: TenantModel): Promise<SubProjectModel[]> {

        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            // Check if user is a tenant admin
            await Auth.isUserAuthorized(
Diego Molteni's avatar
Diego Molteni committed
354
355
                req.headers.authorization, [TenantGroups.adminGroup(tenant)],
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        }

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // Retrieve the subproject list
        const subprojects = await SubProjectDAO.list(journalClient, tenant.name);

        // check If legal tag is valid or remove from the list
        const results: SubProjectModel[] = [];
        const validatedLtag: string[] = [];
        for (const subproject of subprojects) {
            if (subproject.ltag && (FeatureFlags.isEnabled(Feature.LEGALTAG))) {
                // [TODO] we should always have ltag. some datasets does not have it (the old ones)
                if (validatedLtag.indexOf(subproject.ltag) !== -1) {
                    results.push(subproject);
                } else if (await Auth.isLegalTagValid(req.headers.authorization, subproject.ltag,
Diego Molteni's avatar
Diego Molteni committed
373
                    tenant.esd, req[Config.DE_FORWARD_APPKEY], false)) {
Diego Molteni's avatar
Diego Molteni committed
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
                    validatedLtag.push(subproject.ltag);
                    results.push(subproject);
                }
            } else {
                results.push(subproject);
            }
        }

        return results;

    }

    private static async getBucketName(tenant: TenantModel): Promise<string> {
        const storage = StorageFactory.build(Config.CLOUDPROVIDER, tenant);
        for (let i = 0; i < 5; i++) {
            const bucketName = storage.randomBucketName();
            const bucketExists = await storage.bucketExists(bucketName);

            if (!bucketExists) {
                return bucketName;
            }
        }
        throw (Error.make(Error.Status.UNKNOWN, 'Unable to generate a bucket name'));
    }
398
399
400
401
402


    private static validateGroupNamesLength(adminGroupName: string, viewerGroupName: string,
        subproject: SubProjectModel): boolean {

403
        const allowedSubprojectLen = Config.DES_GROUP_CHAR_LIMIT - Math.max(
404
405
406
            adminGroupName.length, viewerGroupName.length
        );

407
        if (allowedSubprojectLen < 0) {
408
409
            throw (Error.make(Error.Status.BAD_REQUEST,
                subproject.name + ' subproject name is too long, for tenant ' + subproject.tenant +
410
                '. The subproject name must not more than ' + Math.abs(allowedSubprojectLen) + ' characters'));
411
412
        }

413
        return true;
414
415
416

    }

417
418
419
420
421
422
423
424
    private static validateAccessPolicy(userInputAccessPolicy: string, exisitngAccessPolicy: string) {

        if (exisitngAccessPolicy === 'dataset' && userInputAccessPolicy === 'uniform') {
            throw (Error.make(Error.Status.BAD_REQUEST, 'Subproject access policy cannot be changed from dataset level to uniform level'));
        }

    }

425

Diego Molteni's avatar
Diego Molteni committed
426
}