handler.ts 47.5 KB
Newer Older
Diego Molteni's avatar
Diego Molteni committed
1
// ============================================================================
2
// Copyright 2017-2021, Schlumberger
Diego Molteni's avatar
Diego Molteni committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ============================================================================

import { Request as expRequest, Response as expResponse } from 'express';
18
19
import { v4 as uuidv4 } from 'uuid';
import { DatasetModel } from '.';
Diego Molteni's avatar
Diego Molteni committed
20
import { Auth } from '../../auth';
21
import { Config, JournalFactoryTenantClient, LoggerFactory, StorageFactory } from '../../cloud';
Diego Molteni's avatar
Diego Molteni committed
22
import { DESStorage, DESUtils } from '../../dataecosystem';
Diego Molteni's avatar
Diego Molteni committed
23
import { Error, Feature, FeatureFlags, Params, Response, Utils } from '../../shared';
24
import { SubProjectDAO, SubProjectModel } from '../subproject';
Diego Molteni's avatar
Diego Molteni committed
25
26
import { TenantDAO, TenantModel } from '../tenant';
import { DatasetDAO } from './dao';
27
import { IWriteLockSession, Locker } from './locker';
Diego Molteni's avatar
Diego Molteni committed
28
29
30
31
32
33
import { DatasetOP } from './optype';
import { DatasetParser } from './parser';
export class DatasetHandler {

    // handler for the [ /dataset ] endpoints
    public static async handler(req: expRequest, res: expResponse, op: DatasetOP) {
Diego Molteni's avatar
Diego Molteni committed
34

Diego Molteni's avatar
Diego Molteni committed
35
        try {
36
37
38
            const tenant = await TenantDAO.get(req.params.tenantid);
            const subproject = await SubProjectDAO.get(
                JournalFactoryTenantClient.get(tenant), req.params.tenantid, req.params.subprojectid);
Diego Molteni's avatar
Diego Molteni committed
39
40

            if (op === DatasetOP.CheckCTag) {
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
                Response.writeOK(res, await this.checkCTag(req, subproject));
            } else if (op === DatasetOP.Register) {
                Response.writeOK(res, await this.register(req, tenant, subproject));
            } else if (op === DatasetOP.Get) {
                Response.writeOK(res, await this.get(req, tenant, subproject));
            } else if (op === DatasetOP.List) {
                Response.writeOK(res, await this.list(req, tenant, subproject));
            } else if (op === DatasetOP.Delete) {
                Response.writeOK(res, await this.delete(req, tenant, subproject));
            } else if (op === DatasetOP.Patch) {
                Response.writeOK(res, await this.patch(req, tenant, subproject));
            } else if (op === DatasetOP.Lock) {
                Response.writeOK(res, await this.lock(req, tenant, subproject));
            } else if (op === DatasetOP.UnLock) {
                Response.writeOK(res, await this.unlock(req, tenant, subproject));
            } else if (op === DatasetOP.Exists) {
                Response.writeOK(res, await this.exists(req, tenant, subproject));
            } else if (op === DatasetOP.Sizes) {
                Response.writeOK(res, await this.sizes(req, tenant, subproject));
            } else if (op === DatasetOP.Permission) {
                Response.writeOK(res, await this.checkPermissions(req, tenant, subproject));
            } else if (op === DatasetOP.ListContent) {
                Response.writeOK(res, await this.listContent(req, tenant, subproject));
            } else if (op === DatasetOP.PutTags) {
                Response.writeOK(res, await this.putTags(req, tenant, subproject));
            } else { throw (Error.make(Error.Status.UNKNOWN, 'Internal Server Error')); }
Diego Molteni's avatar
Diego Molteni committed
67

68
69
70
        } catch (error) {
            Response.writeError(res, error);
        }
Diego Molteni's avatar
Diego Molteni committed
71
72
73

    }

74
    private static async checkCTag(req: expRequest, subproject: SubProjectModel): Promise<boolean> {
Diego Molteni's avatar
Diego Molteni committed
75
76
77
78
79
80

        // parse user request
        const userInput = DatasetParser.checkCTag(req);

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get({
Diego Molteni's avatar
Diego Molteni committed
81
82
            gcpid: userInput.tenantID, esd: userInput.dataPartitionID, default_acls: 'any', name: 'any'
        });
Diego Molteni's avatar
Diego Molteni committed
83

84
85
86
        const datasetOUT = subproject.enforce_key ?
            await DatasetDAO.getByKey(journalClient, userInput.dataset) :
            (await DatasetDAO.get(journalClient, userInput.dataset))[0];
Diego Molteni's avatar
Diego Molteni committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100

        // check if the dataset does not exist
        if (!datasetOUT) {
            throw (Error.make(Error.Status.NOT_FOUND, 'The dataset ' +
                Config.SDPATHPREFIX + userInput.dataset.tenant + '/' +
                userInput.dataset.subproject + userInput.dataset.path +
                userInput.dataset.name + ' does not exist'));
        }

        // return the cTag check validation
        return datasetOUT.ctag === userInput.dataset.ctag;
    }

    // register a new dataset
101
    private static async register(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
102
103

        // parse the user input and create the dataset metadata model
104
        const userInput = await DatasetParser.register(req);
Diego Molteni's avatar
Diego Molteni committed
105
        const dataset = userInput[0];
Diego Molteni's avatar
fixed    
Diego Molteni committed
106
        const seismicmeta = userInput[1];
Diego Molteni's avatar
Diego Molteni committed
107
        let writeLockSession: IWriteLockSession;
Diego Molteni's avatar
Diego Molteni committed
108

Diego Molteni's avatar
Diego Molteni committed
109
        const journalClient = JournalFactoryTenantClient.get(tenant);
Diego Molteni's avatar
Diego Molteni committed
110

Diego Molteni's avatar
Diego Molteni committed
111
        try {
Diego Molteni's avatar
Diego Molteni committed
112

113
114
115
116
            if (dataset.acls) {
                const subprojectMetadata = await SubProjectDAO.get(journalClient, tenant.name, subproject.name);
                const subprojectAccessPolicy = subprojectMetadata.access_policy;

117
                if (subprojectAccessPolicy === Config.UNIFORM_ACCESS_POLICY) {
118
                    throw Error.make(Error.Status.BAD_REQUEST,
119
                        'Subproject access policy is set to uniform and so acls cannot be applied. Patch the subproject access policy to dataset and attempt this operation again.');
120
121
122
                }
            }

Diego Molteni's avatar
Diego Molteni committed
123
            // attempt to acquire a mutex on the dataset name and set the lock for the dataset in redis
124
            // a mutex is applied on the resource on the shared cache (removed at the end of the method)
Diego Molteni's avatar
Diego Molteni committed
125
            const datasetLockKey = dataset.tenant + '/' + dataset.subproject + dataset.path + dataset.name;
Diego Molteni's avatar
Diego Molteni committed
126
            writeLockSession = await Locker.createWriteLock(
Diego Molteni's avatar
Diego Molteni committed
127
                datasetLockKey, req.headers['x-seismic-dms-lockid'] as string);
Diego Molteni's avatar
Diego Molteni committed
128

Diego Molteni's avatar
Diego Molteni committed
129
            // if the call is idempotent return the dataset value
130
            if (writeLockSession.idempotent) {
131
132
133
                const alreadyRegisteredDataset = subproject.enforce_key ?
                    await DatasetDAO.getByKey(journalClient, dataset) :
                    (await DatasetDAO.get(journalClient, dataset))[0];
134
135
136
137
                if (alreadyRegisteredDataset) {
                    await Locker.removeWriteLock(writeLockSession, true); // Keep the lock session
                    return alreadyRegisteredDataset;
                }
Diego Molteni's avatar
Diego Molteni committed
138
            }
Diego Molteni's avatar
Diego Molteni committed
139

140
            // set gcs URL and LegalTag with the subproject information
141
            dataset.gcsurl = subproject.gcs_bucket + '/' + uuidv4();
Diego Molteni's avatar
Diego Molteni committed
142
143
            dataset.ltag = dataset.ltag || subproject.ltag;

Diego Molteni's avatar
fixed    
Diego Molteni committed
144
145
146
147
148
            // ensure that a legal tag exist
            if (!dataset.ltag) {
                throw Error.make(Error.Status.NOT_FOUND,
                    'No legal-tag has been found for the subproject resource ' +
                    Config.SDPATHPREFIX + dataset.tenant + '/' + dataset.subproject +
149
                    ' the storage metadata cannot be updated without a valid a legal-tag');
Diego Molteni's avatar
fixed    
Diego Molteni committed
150
            }
Diego Molteni's avatar
Diego Molteni committed
151

Diego Molteni's avatar
fixed    
Diego Molteni committed
152
153
154
155
156
            // check if has read access, if legal tag is valid, and if the dataset does not already exist
            await Promise.all([
                FeatureFlags.isEnabled(Feature.AUTHORIZATION) ?
                    Auth.isWriteAuthorized(req.headers.authorization,
                        subproject.acls.admins,
Diego Molteni's avatar
Diego Molteni committed
157
                        tenant, dataset.subproject, req[Config.DE_FORWARD_APPKEY]) : undefined,
Diego Molteni's avatar
fixed    
Diego Molteni committed
158
159
160
161
162
                FeatureFlags.isEnabled(Feature.LEGALTAG) ?
                    dataset.ltag ? Auth.isLegalTagValid(
                        req.headers.authorization, dataset.ltag,
                        tenant.esd, req[Config.DE_FORWARD_APPKEY]) : undefined : undefined,
            ]);
Diego Molteni's avatar
Diego Molteni committed
163

164
165
166
167
            const datasetAlreadyExist = subproject.enforce_key ?
                await DatasetDAO.getByKey(journalClient, dataset) :
                (await DatasetDAO.get(journalClient, dataset))[0];

Diego Molteni's avatar
Diego Molteni committed
168
            // check if dataset already exist
169
            if (datasetAlreadyExist) {
Diego Molteni's avatar
Diego Molteni committed
170
                throw (Error.make(Error.Status.ALREADY_EXISTS,
Diego Molteni's avatar
Diego Molteni committed
171
172
173
                    'The dataset ' + Config.SDPATHPREFIX + dataset.tenant + '/' +
                    dataset.subproject + dataset.path + dataset.name +
                    ' already exists'));
Diego Molteni's avatar
Diego Molteni committed
174
175
            }

Diego Molteni's avatar
fixed    
Diego Molteni committed
176
177
            // Populate the storage record with other mandatory field if not supplied.
            if (seismicmeta) {
Diego Molteni's avatar
Diego Molteni committed
178

Diego Molteni's avatar
fixed    
Diego Molteni committed
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
                // if id is given, take it. otherwise generate
                if (!seismicmeta.id) {
                    dataset.seismicmeta_guid = DESUtils.getDataPartitionID(tenant.esd) + seismicmeta.recordType
                        + Utils.makeID(16);
                    seismicmeta.id = dataset.seismicmeta_guid;
                } else {
                    dataset.seismicmeta_guid = seismicmeta.id;
                }

                // remove the recordType attribute as guid is now computed
                delete seismicmeta.recordType;

                // if acl is given, take it. otherwise generate
                if (!seismicmeta.acl) {
                    seismicmeta.acl = {
                        owners: ['data.default.owners@' + tenant.esd],
                        viewers: ['data.default.viewers@' + tenant.esd],
                    };
                }

                // [TO REVIEW]
                // wrt legaltags, there is a field 'otherRelevantDataCountries' that will have to considered
                // for now force it to US, if does not exist. To review before complete PR

                // this could be included as default in the request
                if (!seismicmeta.legal) {
                    seismicmeta.legal = {
                        legaltags: [dataset.ltag],
                        otherRelevantDataCountries: ['US'],
                    };
                }

            }
Diego Molteni's avatar
Diego Molteni committed
212
213


Diego Molteni's avatar
Diego Molteni committed
214
            // prepare the keys
215
            const datasetEntityKey = journalClient.createKey({
Diego Molteni's avatar
Diego Molteni committed
216
217
                namespace: Config.SEISMIC_STORE_NS + '-' + dataset.tenant + '-' + dataset.subproject,
                path: [Config.DATASETS_KIND],
218
                enforcedKey: subproject.enforce_key ? (dataset.path.slice(0, -1) + '/' + dataset.name) : undefined
Diego Molteni's avatar
Diego Molteni committed
219
            });
Diego Molteni's avatar
Diego Molteni committed
220

Diego Molteni's avatar
fixed    
Diego Molteni committed
221
222
            // save the dataset entity
            await Promise.all([
223
                DatasetDAO.register(journalClient, { key: datasetEntityKey, data: dataset }),
Diego Molteni's avatar
fixed    
Diego Molteni committed
224
225
226
227
                (seismicmeta && (FeatureFlags.isEnabled(Feature.SEISMICMETA_STORAGE))) ?
                    DESStorage.insertRecord(req.headers.authorization,
                        [seismicmeta], tenant.esd, req[Config.DE_FORWARD_APPKEY]) : undefined,
            ]);
Diego Molteni's avatar
Diego Molteni committed
228
229
230


            // attach the gcpid for fast check
Diego Molteni's avatar
Diego Molteni committed
231
            dataset.ctag = dataset.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
Diego Molteni's avatar
Diego Molteni committed
232

Diego Molteni's avatar
Diego Molteni committed
233
234
235
236
            // attach locking information
            dataset.sbit = writeLockSession.wid;
            dataset.sbit_count = 1;

Diego Molteni's avatar
Diego Molteni committed
237
            // release the mutex and keep the lock session
Diego Molteni's avatar
Diego Molteni committed
238
            await Locker.removeWriteLock(writeLockSession, true);
Diego Molteni's avatar
Diego Molteni committed
239
            return dataset;
Diego Molteni's avatar
Diego Molteni committed
240

Diego Molteni's avatar
Diego Molteni committed
241
        } catch (err) {
Diego Molteni's avatar
Diego Molteni committed
242

Diego Molteni's avatar
Diego Molteni committed
243
            // release the mutex and unlock the resource
Diego Molteni's avatar
Diego Molteni committed
244
245
            await Locker.removeWriteLock(writeLockSession);
            throw (err);
Diego Molteni's avatar
Diego Molteni committed
246

Diego Molteni's avatar
Diego Molteni committed
247
        }
Diego Molteni's avatar
Diego Molteni committed
248
249
250
251

    }

    // retrieve the dataset metadata
252
    private static async get(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
253
254
255
256
257

        // parse user request
        const userInput = DatasetParser.get(req);
        const datasetIN = userInput[0];

Diego Molteni's avatar
Diego Molteni committed
258
        // retrieve journalClient client
Diego Molteni's avatar
Diego Molteni committed
259
260
261
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // Retrieve the dataset metadata
262
263
264
        const datasetOUT = subproject.enforce_key ?
            await DatasetDAO.getByKey(journalClient, datasetIN) :
            (await DatasetDAO.get(journalClient, datasetIN))[0];
Diego Molteni's avatar
Diego Molteni committed
265

Diego Molteni's avatar
Diego Molteni committed
266
267
268
269
270
271
272
273
274
275
        // check if the dataset does not exist
        if (!datasetOUT) {
            throw (Error.make(Error.Status.NOT_FOUND,
                'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
        }

        // Check if retrieve the seismic metadata storage record
        const getSeismicMeta = datasetOUT.seismicmeta_guid !== undefined && userInput[1];

276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
        // Check if legal tag is valid
        if (FeatureFlags.isEnabled(Feature.LEGALTAG) && datasetOUT.ltag) {
            await Auth.isLegalTagValid(req.headers.authorization, datasetOUT.ltag,
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }


        // Use the access policy to determine which groups to fetch for read authorization
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            let authGroups = [];
            if (subproject.access_policy === Config.UNIFORM_ACCESS_POLICY) {
                authGroups = subproject.acls.viewers.concat(subproject.acls.admins);
            } else if (subproject.access_policy === Config.DATASET_ACCESS_POLICY) {
                authGroups = datasetOUT.acls ? datasetOUT.acls.viewers.concat(datasetOUT.acls.admins)
                    : subproject.acls.viewers.concat(subproject.acls.admins);
            } else {
Diego Molteni's avatar
Diego Molteni committed
292
293
                throw (Error.make(Error.Status.PERMISSION_DENIED,
                    'Access policy for the subproject is neither uniform nor dataset'));
294
295
            }
            await Auth.isReadAuthorized(req.headers.authorization, authGroups,
Diego Molteni's avatar
Diego Molteni committed
296
                tenant, datasetIN.subproject, req[Config.DE_FORWARD_APPKEY]);
297
        }
Diego Molteni's avatar
Diego Molteni committed
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315

        // return the seismicmetadata (if exist)
        if (getSeismicMeta) {
            const seismicMeta = await DESStorage.getRecord(req.headers.authorization, datasetOUT.seismicmeta_guid,
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
            if (seismicMeta) {
                (datasetOUT as any).seismicmeta = seismicMeta;
            }
        }

        // attach the gcpid for fast check
        datasetOUT.ctag = datasetOUT.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);

        return datasetOUT;

    }

    // list the datasets in a subproject
316
    private static async list(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
317
318
319
320

        // Retrieve the dataset path information
        const dataset = DatasetParser.list(req);

Diego Molteni's avatar
Diego Molteni committed
321
322
323
        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

Diego Molteni's avatar
Diego Molteni committed
324
325
326
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            // Check authorizations
            await Auth.isReadAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
327
                subproject.acls.viewers.concat(subproject.acls.admins),
Diego Molteni's avatar
Diego Molteni committed
328
                tenant, dataset.subproject, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
        }


        // Retrieve the list of datasets metadata
        const datasets = await DatasetDAO.list(journalClient, dataset);

        // attach the gcpid for fast check
        for (const item of datasets) {
            item.ctag = item.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
        }

        // Retrieve the list of datasets metadata
        return datasets;

    }

    // delete a dataset
346
    private static async delete(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
347
348
349

        // Retrieve the dataset path information
        const datasetIn = DatasetParser.delete(req);
Diego Molteni's avatar
Diego Molteni committed
350
        const lockKey = datasetIn.tenant + '/' + datasetIn.subproject + datasetIn.path + datasetIn.name;
Diego Molteni's avatar
Diego Molteni committed
351

Diego Molteni's avatar
Diego Molteni committed
352
        // ensure is not write locked
353
        if (!Config.SKIP_WRITE_LOCK_CHECK_ON_MUTABLE_OPERATIONS) {
Diego Molteni's avatar
Diego Molteni committed
354
            if (Locker.isWriteLock(await Locker.getLock(lockKey))) {
Diego Molteni's avatar
Diego Molteni committed
355
356
                throw (Error.make(Error.Status.LOCKED,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIn.tenant + '/' +
357
358
                    datasetIn.subproject + datasetIn.path + datasetIn.name + ' is write locked ' +
                    Error.get423WriteLockReason()));
Diego Molteni's avatar
Diego Molteni committed
359
360
            }
        }
Diego Molteni's avatar
Diego Molteni committed
361
362

        // init datastore client
Diego Molteni's avatar
Diego Molteni committed
363
        const journalClient = JournalFactoryTenantClient.get(tenant);
Diego Molteni's avatar
Diego Molteni committed
364

Diego Molteni's avatar
Diego Molteni committed
365
        // Retrieve the dataset metadata
366
367
368
        const dataset = subproject.enforce_key ?
            await DatasetDAO.getByKey(journalClient, datasetIn) :
            (await DatasetDAO.get(journalClient, datasetIn))[0];
Diego Molteni's avatar
Diego Molteni committed
369

Diego Molteni's avatar
Diego Molteni committed
370
371
        // if the dataset does not exist return ok
        if (!dataset) { return; }
Diego Molteni's avatar
Diego Molteni committed
372

373
374
375
376
377
378
379
380
381
382
        // check authorization (write)
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            let authGroups = [];
            const accessPolicy = subproject.access_policy;

            if (accessPolicy === Config.UNIFORM_ACCESS_POLICY) {
                authGroups = subproject.acls.admins;
            } else if (accessPolicy === Config.DATASET_ACCESS_POLICY) {
                authGroups = dataset.acls ? dataset.acls.admins : subproject.acls.admins;
            } else {
Diego Molteni's avatar
Diego Molteni committed
383
384
                throw (Error.make(Error.Status.PERMISSION_DENIED,
                    'Access policy for the subproject is neither uniform nor dataset'));
385
386
387
            }

            await Auth.isWriteAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
388
                authGroups, tenant, subproject.name, req[Config.DE_FORWARD_APPKEY]);
389
390
        }

Diego Molteni's avatar
Diego Molteni committed
391
392
393
394
395
396
397
        // check if valid url
        if (!dataset.gcsurl || dataset.gcsurl.indexOf('/') === -1) {
            throw (Error.make(Error.Status.UNKNOWN,
                'The dataset ' + Config.SDPATHPREFIX + datasetIn.tenant + '/' +
                datasetIn.subproject + datasetIn.path + datasetIn.name +
                ' cannot be deleted as it does not have a valid gcs url in the metadata catalogue.'));
        }
Diego Molteni's avatar
Diego Molteni committed
398

Diego Molteni's avatar
Diego Molteni committed
399
400
401
402
403
404
405
406
407
        // Delete the dataset metadata (both firestore and DEStorage)
        await Promise.all([
            // delete the dataset entity
            DatasetDAO.delete(journalClient, dataset),
            // delete des storage record
            (dataset.seismicmeta_guid && (FeatureFlags.isEnabled(Feature.SEISMICMETA_STORAGE))) ?
                DESStorage.deleteRecord(req.headers.authorization,
                    dataset.seismicmeta_guid, tenant.esd, req[Config.DE_FORWARD_APPKEY]) : undefined,
        ]);
Diego Molteni's avatar
Diego Molteni committed
408

409
        // Delete all physical objects (not wait for full objects deletion)
Diego Molteni's avatar
Diego Molteni committed
410
        const bucketName = dataset.gcsurl.split('/')[0];
411
        const gcsPrefix = dataset.gcsurl.split('/')[1];
Diego Molteni's avatar
Diego Molteni committed
412
        const storage = StorageFactory.build(Config.CLOUDPROVIDER, tenant);
413
        // tslint:disable-next-line: no-floating-promises no-console
414
415
416
        storage.deleteObjects(bucketName, gcsPrefix).catch((error) => {
            LoggerFactory.build(Config.CLOUDPROVIDER).error(JSON.stringify(error));
        });
Diego Molteni's avatar
Diego Molteni committed
417

Diego Molteni's avatar
Diego Molteni committed
418
419
        // remove any remaining locks (this should be removed with SKIP_WRITE_LOCK_CHECK_ON_MUTABLE_OPERATIONS)
        const datasetLockKey = dataset.tenant + '/' + dataset.subproject + dataset.path + dataset.name;
420
        await Locker.unlock(datasetLockKey);
Diego Molteni's avatar
Diego Molteni committed
421

Diego Molteni's avatar
Diego Molteni committed
422
423
424
    }

    // patch the dataset metadata
425
    private static async patch(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
426
427
428

        // Retrieve the dataset path information
        const [datasetIN, seismicmeta, newName, wid] = DatasetParser.patch(req);
Diego Molteni's avatar
Diego Molteni committed
429
        const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
430

Diego Molteni's avatar
Diego Molteni committed
431
        // retrieve datastore client
Diego Molteni's avatar
Diego Molteni committed
432
        const journalClient = JournalFactoryTenantClient.get(tenant);
Diego Molteni's avatar
Diego Molteni committed
433

434
        // return immediately if it is a simple close with empty body (no patch to apply)
435
436
        if (Object.keys(req.body).length === 0 && req.body.constructor === Object && wid) {

437
438
            // Check authorizations
            if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
439
                if (wid.startsWith('W')) {
440
441
                    await Auth.isWriteAuthorized(req.headers.authorization,
                        subproject.acls.admins,
Diego Molteni's avatar
Diego Molteni committed
442
                        tenant, subproject.name, req[Config.DE_FORWARD_APPKEY]);
443
444
445
                } else {
                    await Auth.isReadAuthorized(req.headers.authorization,
                        subproject.acls.viewers.concat(subproject.acls.admins),
Diego Molteni's avatar
Diego Molteni committed
446
                        tenant, datasetIN.subproject, req[Config.DE_FORWARD_APPKEY]);
447
448
449
                }
            }

450
            // Retrieve the dataset metadata
451
452
453
            const dataset = subproject.enforce_key ?
                await DatasetDAO.getByKey(journalClient, datasetIN) :
                (await DatasetDAO.get(journalClient, datasetIN))[0];
454
455
456
457
458
459
460
461

            // check if the dataset does not exist
            if (!dataset) {
                throw (Error.make(Error.Status.NOT_FOUND,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                    datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
            }

462
            // unlock the dataset
463
            const unlockRes = await Locker.unlock(lockKey, wid);
Diego Molteni's avatar
Diego Molteni committed
464
465
            dataset.sbit = unlockRes.id;
            dataset.sbit_count = unlockRes.cnt;
466
467
468
469

            return dataset;
        }

470
471
472
473
474
        // Ensure subproject access policy is not set to uniform
        if (datasetIN.acls) {
            const subprojectMetadata = await SubProjectDAO.get(journalClient, tenant.name, subproject.name);
            const subprojectAccessPolicy = subprojectMetadata.access_policy;

475
            if (subprojectAccessPolicy === Config.UNIFORM_ACCESS_POLICY) {
476
477
478
479
480
                throw Error.make(Error.Status.BAD_REQUEST,
                    'Subproject access policy is set to uniform and so the dataset acls cannot be applied. Patch the subproject access policy to dataset and attempt this operation again.');
            }
        }

481
        // unlock the dataset for close operation (and patch)
Diego Molteni's avatar
Diego Molteni committed
482
        const lockres = wid ? await Locker.unlock(lockKey, wid) : { id: null, cnt: 0 };
Diego Molteni's avatar
Diego Molteni committed
483

484
        // ensure nobody got the lock between the close and the mutex acquisition
485
        if (!Config.SKIP_WRITE_LOCK_CHECK_ON_MUTABLE_OPERATIONS) {
Diego Molteni's avatar
Diego Molteni committed
486
            if (Locker.isWriteLock(await Locker.getLock(lockKey))) {
Diego Molteni's avatar
Diego Molteni committed
487
488
                throw (Error.make(Error.Status.LOCKED,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
489
490
                    datasetIN.subproject + datasetIN.path + datasetIN.name + ' is write locked ' +
                    Error.get423WriteLockReason()));
Diego Molteni's avatar
Diego Molteni committed
491
492
            }
        }
Diego Molteni's avatar
Diego Molteni committed
493

Diego Molteni's avatar
Diego Molteni committed
494
        // Retrieve the dataset metadata
495
496
        let datasetOUT: DatasetModel;
        let datasetOUTKey: any;
497
        if (subproject.enforce_key) {
498
499
            datasetOUT = await DatasetDAO.getByKey(journalClient, datasetIN);
            datasetOUTKey = journalClient.createKey({
500
501
502
503
                namespace: Config.SEISMIC_STORE_NS + '-' + datasetIN.tenant + '-' + datasetIN.subproject,
                path: [Config.DATASETS_KIND],
                enforcedKey: datasetIN.path.slice(0, -1) + '/' + datasetIN.name
            });
504
505
506
507
508
        } else {
            const results = await DatasetDAO.get(journalClient, datasetIN);
            datasetOUT = results[0];
            datasetOUTKey = results[1];
        }
Diego Molteni's avatar
Diego Molteni committed
509

Diego Molteni's avatar
Diego Molteni committed
510
511
512
513
514
515
        // check if the dataset does not exist
        if (!datasetOUT) {
            throw (Error.make(Error.Status.NOT_FOUND,
                'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
        }
Diego Molteni's avatar
Diego Molteni committed
516

517
        // If the input request has dataset acls then the subproject access policy is always dataset
Diego Molteni's avatar
Diego Molteni committed
518
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
519
520
521
522
523
524
525
526
527
528
529
530
            let authGroups = [];
            const accessPolicy = subproject.access_policy;

            if (accessPolicy === Config.UNIFORM_ACCESS_POLICY) {
                authGroups = subproject.acls.admins;
            } else if (accessPolicy === Config.DATASET_ACCESS_POLICY) {
                authGroups = datasetOUT.acls ? datasetOUT.acls.admins : subproject.acls.admins;
            } else {
                throw (Error.make(Error.Status.PERMISSION_DENIED, 'Access policy is neither uniform nor dataset.'
                ));
            }

Diego Molteni's avatar
Diego Molteni committed
531
            await Auth.isWriteAuthorized(req.headers.authorization,
532
                authGroups, tenant, subproject.name, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
533
534
535
536
537
538
539
540
541
542
543
544
        }

        // patch datasetOUT with datasetIN
        if (datasetIN.metadata) { datasetOUT.metadata = datasetIN.metadata; }
        if (datasetIN.filemetadata) { datasetOUT.filemetadata = datasetIN.filemetadata; }
        if (datasetIN.last_modified_date) { datasetOUT.last_modified_date = datasetIN.last_modified_date; }
        if (datasetIN.readonly !== undefined) { datasetOUT.readonly = datasetIN.readonly; }
        if (datasetIN.gtags !== undefined && datasetIN.gtags.length > 0) { datasetOUT.gtags = datasetIN.gtags; }
        if (datasetIN.ltag) {
            if (FeatureFlags.isEnabled(Feature.LEGALTAG)) {
                await Auth.isLegalTagValid(
                    req.headers.authorization, datasetIN.ltag, tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
fixed    
Diego Molteni committed
545
            }
Diego Molteni's avatar
Diego Molteni committed
546
547
            datasetOUT.ltag = datasetIN.ltag;
        }
Diego Molteni's avatar
Diego Molteni committed
548

Diego Molteni's avatar
Diego Molteni committed
549
550
551
552
553
554
        if (newName) {
            if (newName === datasetIN.name) {
                throw (Error.make(Error.Status.ALREADY_EXISTS,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                    datasetIN.subproject + datasetIN.path + newName + ' already exists'));
            }
Diego Molteni's avatar
Diego Molteni committed
555

Diego Molteni's avatar
Diego Molteni committed
556
            datasetIN.name = newName;
Diego Molteni's avatar
Diego Molteni committed
557

558
559
560
            const datasetAlreadyExist = subproject.enforce_key ?
                await DatasetDAO.getByKey(journalClient, datasetIN) :
                (await DatasetDAO.get(journalClient, datasetIN))[0];
Diego Molteni's avatar
Diego Molteni committed
561

562
563
            // check if dataset already exist
            if (datasetAlreadyExist) {
Diego Molteni's avatar
Diego Molteni committed
564
565
566
                throw (Error.make(Error.Status.ALREADY_EXISTS,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                    datasetIN.subproject + datasetIN.path + newName + ' already exists'));
567
            }
Diego Molteni's avatar
fixed    
Diego Molteni committed
568

569
            if (subproject.enforce_key) {
570
571
572
                datasetOUTKey = journalClient.createKey({
                    namespace: Config.SEISMIC_STORE_NS + '-' + datasetIN.tenant + '-' + datasetIN.subproject,
                    path: [Config.DATASETS_KIND],
573
574
                    enforcedKey: datasetIN.path.slice(0, -1) + '/' + datasetIN.name
                });
Diego Molteni's avatar
Diego Molteni committed
575
            }
Diego Molteni's avatar
fixed    
Diego Molteni committed
576

Diego Molteni's avatar
Diego Molteni committed
577
578
            datasetOUT.name = newName;
        }
Diego Molteni's avatar
Diego Molteni committed
579

Diego Molteni's avatar
Diego Molteni committed
580
581
582
        // Populate the storage record with other mandatory field if not supplied.
        let seismicmetaDE: any;
        if (seismicmeta) {
Diego Molteni's avatar
Diego Molteni committed
583

Diego Molteni's avatar
Diego Molteni committed
584
585
            // return the seismicmetadata (if exists)
            if (datasetOUT.seismicmeta_guid) {
Diego Molteni's avatar
Diego Molteni committed
586

Diego Molteni's avatar
Diego Molteni committed
587
588
589
590
                // seismicmeta is already there, need to patch
                seismicmetaDE = await DESStorage.getRecord(
                    req.headers.authorization, datasetOUT.seismicmeta_guid,
                    tenant.esd, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
591

Diego Molteni's avatar
Diego Molteni committed
592
593
594
                for (const keyx of Object.keys(seismicmeta)) {
                    seismicmetaDE[keyx] = seismicmeta[keyx];
                }
Diego Molteni's avatar
fixed    
Diego Molteni committed
595

Diego Molteni's avatar
Diego Molteni committed
596
                datasetOUT.seismicmeta_guid = seismicmeta.id;
Diego Molteni's avatar
fixed    
Diego Molteni committed
597

Diego Molteni's avatar
Diego Molteni committed
598
            } else {
Diego Molteni's avatar
fixed    
Diego Molteni committed
599

Diego Molteni's avatar
Diego Molteni committed
600
601
602
                // mandatory field required if a new seismic metadata record is ingested (kind/data required)
                Params.checkString(seismicmeta.kind, 'kind');
                Params.checkObject(seismicmeta.data, 'data');
Diego Molteni's avatar
fixed    
Diego Molteni committed
603

604
                // {data-partition(delfi)|authority(osdu)}.{source}.{entityType}.{semanticSchemaVersion}
605
                if ((seismicmeta.kind as string).split(':').length !== 4) {
Diego Molteni's avatar
Diego Molteni committed
606
607
                    throw (Error.make(Error.Status.BAD_REQUEST, 'The seismicmeta kind is in a wrong format'));
                }
Diego Molteni's avatar
fixed    
Diego Molteni committed
608

609
                // (recordType == entityType)
Diego Molteni's avatar
Diego Molteni committed
610
                seismicmeta.recordType = ':' + (seismicmeta.kind as string).split(':')[2] + ':';
Diego Molteni's avatar
fixed    
Diego Molteni committed
611

Diego Molteni's avatar
Diego Molteni committed
612
613
614
615
616
                // if id is given, take it. otherwise generate
                if (!seismicmeta.id) {
                    datasetOUT.seismicmeta_guid = DESUtils.getDataPartitionID(tenant.esd)
                        + seismicmeta.recordType + Utils.makeID(16);
                    seismicmeta.id = datasetOUT.seismicmeta_guid;
Diego Molteni's avatar
fixed    
Diego Molteni committed
617
                } else {
Diego Molteni's avatar
Diego Molteni committed
618
619
                    datasetOUT.seismicmeta_guid = seismicmeta.id;
                }
Diego Molteni's avatar
fixed    
Diego Molteni committed
620

Diego Molteni's avatar
Diego Molteni committed
621
622
                // remove the recordType attribute as guid is now computed
                delete seismicmeta.recordType;
Diego Molteni's avatar
fixed    
Diego Molteni committed
623

Diego Molteni's avatar
Diego Molteni committed
624
625
626
627
628
629
630
                // if acl is given, take it. otherwise generate
                if (!seismicmeta.acl) {
                    seismicmeta.acl = {
                        owners: ['data.default.owners@' + tenant.esd],
                        viewers: ['data.default.viewers@' + tenant.esd],
                    };
                }
Diego Molteni's avatar
fixed    
Diego Molteni committed
631

Diego Molteni's avatar
Diego Molteni committed
632
633
634
                // [TO REVIEW]
                // wrt legaltags, there is a field 'otherRelevantDataCountries' that will have to considered
                // for now force it to US, if does not exist. To review before complete PR
Diego Molteni's avatar
fixed    
Diego Molteni committed
635

Diego Molteni's avatar
Diego Molteni committed
636
637
                // this could be included as default in the request
                if (!seismicmeta.legal) {
Diego Molteni's avatar
fixed    
Diego Molteni committed
638

Diego Molteni's avatar
Diego Molteni committed
639
640
641
642
643
644
645
                    // ensure that a legal tag exist
                    if (!datasetOUT.ltag) {

                        throw (!subproject.ltag ?
                            Error.make(Error.Status.NOT_FOUND,
                                'No legal-tag has been found for the subproject resource ' +
                                Config.SDPATHPREFIX + datasetIN.tenant + '/' + datasetIN.subproject +
646
                                ' the storage metadata cannot be updated without a valid legal-tag') :
Diego Molteni's avatar
Diego Molteni committed
647
648
649
650
                            Error.make(Error.Status.NOT_FOUND,
                                'No legal-tag has been found on the dataset resource ' +
                                Config.SDPATHPREFIX + datasetIN.tenant + '/' + datasetIN.subproject +
                                datasetIN.path + datasetIN.name +
651
                                ' the storage metadata cannot be updated without a valid legal-tag'));
Diego Molteni's avatar
fixed    
Diego Molteni committed
652
653
                    }

Diego Molteni's avatar
Diego Molteni committed
654
655
656
657
658
                    // insert legal tag
                    seismicmeta.legal = {
                        legaltags: [datasetOUT.ltag],
                        otherRelevantDataCountries: ['US'],
                    };
Diego Molteni's avatar
fixed    
Diego Molteni committed
659
                }
Diego Molteni's avatar
Diego Molteni committed
660
661

                seismicmetaDE = seismicmeta;
Diego Molteni's avatar
fixed    
Diego Molteni committed
662
            }
Diego Molteni's avatar
Diego Molteni committed
663
        }
Diego Molteni's avatar
Diego Molteni committed
664

665
666
667
668
669
        // Update the acls if the input request has them
        if (datasetIN.acls) {
            datasetOUT.acls = datasetIN.acls;
        }

670
        if (newName) {
671
672
            await Promise.all([
                DatasetDAO.delete(journalClient, datasetOUT),
673
                DatasetDAO.register(journalClient, { key: datasetOUTKey, data: datasetOUT }),
674
675
676
677
678
679
680
681
682
683
                (seismicmeta && (FeatureFlags.isEnabled(Feature.SEISMICMETA_STORAGE)))
                    ? DESStorage.insertRecord(req.headers.authorization, [seismicmetaDE],
                        tenant.esd, req[Config.DE_FORWARD_APPKEY]) : undefined]);
        } else {
            await Promise.all([
                DatasetDAO.update(journalClient, datasetOUT, datasetOUTKey),
                (seismicmeta && (FeatureFlags.isEnabled(Feature.SEISMICMETA_STORAGE)))
                    ? DESStorage.insertRecord(req.headers.authorization, [seismicmetaDE],
                        tenant.esd, req[Config.DE_FORWARD_APPKEY]) : undefined]);
        }
Diego Molteni's avatar
Diego Molteni committed
684
685
686
687
688
689
690
691
692

        // attach lock information
        if (wid) {
            datasetOUT.sbit = lockres.id;
            datasetOUT.sbit_count = lockres.cnt;
        } else {
            const datasetOUTLockKey = datasetOUT.tenant + '/' + datasetOUT.subproject
                + datasetOUT.path + datasetOUT.name;
            const datasetOUTLockRes = await Locker.getLock(datasetOUTLockKey);
693
694
            if (datasetOUTLockRes) {
                if (Locker.isWriteLock(datasetOUTLockRes)) {
Diego Molteni's avatar
Diego Molteni committed
695
696
697
698
699
700
                    datasetOUT.sbit = datasetOUTLockRes as string;
                    datasetOUT.sbit_count = 1;
                } else {
                    datasetOUT.sbit = (datasetOUTLockRes as string[]).join(':');
                    datasetOUT.sbit_count = datasetOUTLockRes.length;
                }
Diego Molteni's avatar
Diego Molteni committed
701
            }
Diego Molteni's avatar
Diego Molteni committed
702
        }
Diego Molteni's avatar
Diego Molteni committed
703

Diego Molteni's avatar
Diego Molteni committed
704
705
        // attach the gcpid for fast check
        datasetOUT.ctag = datasetOUT.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
Diego Molteni's avatar
Diego Molteni committed
706

Diego Molteni's avatar
Diego Molteni committed
707
        return datasetOUT;
Diego Molteni's avatar
Diego Molteni committed
708

Diego Molteni's avatar
Diego Molteni committed
709
710
711
    }

    // lock the dataset metadata for opening
712
    private static async lock(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
713
714
715
716
717
718
719

        // parse user request
        const userInput = DatasetParser.lock(req);
        const datasetIN = userInput.dataset;
        const open4write = userInput.open4write;
        const wid = userInput.wid;

Diego Molteni's avatar
Diego Molteni committed
720
        // retrieve datastore client
Diego Molteni's avatar
Diego Molteni committed
721
722
        const journalClient = JournalFactoryTenantClient.get(tenant);

Diego Molteni's avatar
Diego Molteni committed
723

724
725
726
727
        // Retrieve the dataset metadata
        const datasetOUT = subproject.enforce_key ?
            await DatasetDAO.getByKey(journalClient, datasetIN) :
            (await DatasetDAO.get(journalClient, datasetIN))[0];
Diego Molteni's avatar
Diego Molteni committed
728

Diego Molteni's avatar
Diego Molteni committed
729
730
731
732
733
734
735
        // check if the dataset does not exist
        if (!datasetOUT) {
            throw (Error.make(Error.Status.NOT_FOUND,
                'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
        }

736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
        // Check if legal tag is valid;
        if (FeatureFlags.isEnabled(Feature.LEGALTAG) && datasetOUT.ltag) {
            await Auth.isLegalTagValid(req.headers.authorization, datasetOUT.ltag,
                tenant.esd, req[Config.DE_FORWARD_APPKEY]);
        }

        // Use the access policy to determine which groups to fetch for read authorization
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            let authGroups = [];
            const accessPolicy = subproject.access_policy;

            if (open4write) {
                if (accessPolicy === Config.UNIFORM_ACCESS_POLICY) {
                    authGroups = subproject.acls.admins;
                } else if (accessPolicy === Config.DATASET_ACCESS_POLICY) {
                    authGroups = datasetOUT.acls ? datasetOUT.acls.admins : subproject.acls.admins;
                } else {
Diego Molteni's avatar
Diego Molteni committed
753
754
                    throw (Error.make(Error.Status.PERMISSION_DENIED,
                        'Access policy is neither uniform nor dataset'));
755
756
757
                }

                await Auth.isWriteAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
758
                    authGroups, tenant, datasetIN.subproject, req[Config.DE_FORWARD_APPKEY]);
759
760
761
762
763
764
765
766
767

            } else {

                if (accessPolicy === Config.UNIFORM_ACCESS_POLICY) {
                    authGroups = subproject.acls.viewers.concat(subproject.acls.admins);
                } else if (accessPolicy === Config.DATASET_ACCESS_POLICY) {
                    authGroups = datasetOUT.acls ? datasetOUT.acls.viewers.concat(datasetOUT.acls.admins)
                        : subproject.acls.viewers.concat(subproject.acls.admins);
                } else {
Diego Molteni's avatar
Diego Molteni committed
768
769
                    throw (Error.make(Error.Status.PERMISSION_DENIED,
                        'Access policy is neither uniform nor dataset'));
770
771
772
                }

                await Auth.isReadAuthorized(req.headers.authorization, authGroups,
Diego Molteni's avatar
Diego Molteni committed
773
                    tenant, datasetIN.subproject, req[Config.DE_FORWARD_APPKEY]);
774
775
776
            }

        }
Diego Molteni's avatar
Diego Molteni committed
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792

        // managing read-only datasets
        if (datasetOUT.readonly) {
            if (open4write) {
                throw (Error.make(Error.Status.BAD_REQUEST,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                    datasetIN.subproject + datasetIN.path + datasetIN.name +
                    ' cannot be locked (read-only dataset)'));
            } else {
                // attach the gcpid for fast check
                datasetOUT.ctag = datasetOUT.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);
                return datasetOUT;
            }
        }

        // lock in cache
Diego Molteni's avatar
Diego Molteni committed
793
        const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
Diego Molteni's avatar
Diego Molteni committed
794
        const lockres = open4write ?
Diego Molteni's avatar
Diego Molteni committed
795
796
            await Locker.acquireWriteLock(lockKey, req.headers['x-seismic-dms-lockid'] as string, wid) :
            await Locker.acquireReadLock(lockKey, req.headers['x-seismic-dms-lockid'] as string, wid);
Diego Molteni's avatar
Diego Molteni committed
797
798
799
800
801
802
803
804
805
806
807
808
809

        // attach lock information
        datasetOUT.sbit = lockres.id;
        datasetOUT.sbit_count = lockres.cnt;

        // attach the gcpid for fast check
        datasetOUT.ctag = datasetOUT.ctag + tenant.gcpid + ';' + DESUtils.getDataPartitionID(tenant.esd);

        return datasetOUT;

    }

    // unlock the dataset metadata for opening
810
    private static async unlock(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
811

Diego Molteni's avatar
Diego Molteni committed
812
        // parse user request
Diego Molteni's avatar
Diego Molteni committed
813
814
        const datasetIN = DatasetParser.unlock(req);

Diego Molteni's avatar
Diego Molteni committed
815
        // retrieve datastore client
Diego Molteni's avatar
Diego Molteni committed
816
817
818
        const journalClient = JournalFactoryTenantClient.get(tenant);

        // Retrieve the dataset metadata
819
820
821
        const dataset = subproject.enforce_key ?
            await DatasetDAO.getByKey(journalClient, datasetIN) :
            (await DatasetDAO.get(journalClient, datasetIN))[0];
Diego Molteni's avatar
Diego Molteni committed
822

Diego Molteni's avatar
Diego Molteni committed
823

Diego Molteni's avatar
Diego Molteni committed
824
825
826
827
828
829
830
831
        // check if the dataset does not exist
        if (!dataset) {
            throw (Error.make(Error.Status.NOT_FOUND,
                'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
        }

        // check if user is write authorized
832
833
834
835
836
837
838
839
        let authGroups = [];
        const accessPolicy = subproject.access_policy;

        if (accessPolicy === Config.UNIFORM_ACCESS_POLICY) {
            authGroups = subproject.acls.admins;
        } else if (accessPolicy === Config.DATASET_ACCESS_POLICY) {
            authGroups = dataset.acls ? dataset.acls.admins : subproject.acls.admins;
        } else {
Diego Molteni's avatar
Diego Molteni committed
840
841
            throw (Error.make(Error.Status.PERMISSION_DENIED,
                'Access policy is neither uniform nor dataset'));
842
843
        }

Diego Molteni's avatar
Diego Molteni committed
844
        await Auth.isWriteAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
845
            authGroups, tenant, dataset.subproject, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
846
847

        // unlock
Diego Molteni's avatar
Diego Molteni committed
848
849
        const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
        await Locker.unlock(lockKey);
Diego Molteni's avatar
Diego Molteni committed
850
851
852
853

    }

    // check if a list of datasets exist in a subproject
854
    private static async exists(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
855
856
857
858

        // Retrieve the dataset path information
        const datasets = DatasetParser.exists(req);

Diego Molteni's avatar
Diego Molteni committed
859
860
861
        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

Diego Molteni's avatar
Diego Molteni committed
862
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
863

Diego Molteni's avatar
Diego Molteni committed
864
            await Auth.isReadAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
865
                subproject.acls.viewers.concat(subproject.acls.admins),
Diego Molteni's avatar
Diego Molteni committed
866
                tenant, datasets[0].subproject, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
867
868
869
870
        }

        // Check if the required datasets exist
        const results: boolean[] = [];
871
        if (subproject.enforce_key) {
872
            for (const dataset of datasets) {
873
                results.push((await DatasetDAO.getByKey(journalClient, dataset)) !== undefined);
874
875
876
877
878
            }
        } else {
            for (const dataset of datasets) {
                results.push((await DatasetDAO.get(journalClient, dataset))[0] !== undefined);
            }
Diego Molteni's avatar
Diego Molteni committed
879
880
881
882
883
        }
        return results;
    }

    // retrieve the dataset size for a list of datasets
884
    private static async sizes(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
885
886
887
888

        // Retrieve the dataset path information
        const datasets = DatasetParser.sizes(req);

Diego Molteni's avatar
Diego Molteni committed
889
890
891
        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

Diego Molteni's avatar
Diego Molteni committed
892
893
        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
            await Auth.isReadAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
894
                subproject.acls.viewers.concat(subproject.acls.admins),
Diego Molteni's avatar
Diego Molteni committed
895
                tenant, datasets[0].subproject, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
896
897
898
899
        }

        // Check if the required datasets exist
        const results: number[] = [];
900
        if (subproject.enforce_key) {
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
            for (let dataset of datasets) {
                dataset = await DatasetDAO.getByKey(journalClient, dataset);
                if (dataset === undefined) {
                    results.push(-1);
                    continue;
                }
                results.push(!dataset.filemetadata || !dataset.filemetadata.size ? -1 : dataset.filemetadata.size);
            }
        } else {
            for (let dataset of datasets) {
                dataset = (await DatasetDAO.get(journalClient, dataset))[0];
                if (dataset === undefined) {
                    results.push(-1);
                    continue;
                }
                results.push(!dataset.filemetadata || !dataset.filemetadata.size ? -1 : dataset.filemetadata.size);
Diego Molteni's avatar
Diego Molteni committed
917
918
919
920
921
922
923
924
            }
        }

        return results;

    }

    // list the path content
925
    private static async listContent(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
926
927
928
929

        // Retrieve the dataset information
        const dataset = DatasetParser.listContent(req);

Diego Molteni's avatar
Diego Molteni committed
930
931
932
933
        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

        if (FeatureFlags.isEnabled(Feature.AUTHORIZATION)) {
Diego Molteni's avatar
Diego Molteni committed
934
935
            // Check authorizations
            await Auth.isReadAuthorized(req.headers.authorization,
Diego Molteni's avatar
Diego Molteni committed
936
                subproject.acls.viewers.concat(subproject.acls.admins),
Diego Molteni's avatar
Diego Molteni committed
937
                tenant, dataset.subproject, req[Config.DE_FORWARD_APPKEY]);
Diego Molteni's avatar
Diego Molteni committed
938
939
940
941
942
943
944
        }

        // list the folder content
        return await DatasetDAO.listContent(journalClient, dataset);

    }

945
    private static async putTags(req: expRequest, tenant: TenantModel, subproject: SubProjectModel) {
Diego Molteni's avatar
Diego Molteni committed
946
947
948
949
950
951

        const datasetIN = DatasetParser.putTags(req);

        // init journalClient client
        const journalClient = JournalFactoryTenantClient.get(tenant);

Diego Molteni's avatar
Diego Molteni committed
952
        // ensure is not write locked
953
        if (!Config.SKIP_WRITE_LOCK_CHECK_ON_MUTABLE_OPERATIONS) {
Diego Molteni's avatar
Diego Molteni committed
954
955
            const lockKey = datasetIN.tenant + '/' + datasetIN.subproject + datasetIN.path + datasetIN.name;
            if (Locker.isWriteLock(await Locker.getLock(lockKey))) {
Diego Molteni's avatar
Diego Molteni committed
956
957
                throw (Error.make(Error.Status.LOCKED,
                    'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
958
959
                    datasetIN.subproject + datasetIN.path + datasetIN.name + ' is write locked ' +
                    Error.get423WriteLockReason()));
Diego Molteni's avatar
Diego Molteni committed
960
961
            }
        }
Diego Molteni's avatar
Diego Molteni committed
962

963
964
965
        // Retrieve the dataset metadata
        let datasetOUT: DatasetModel;
        let datasetOUTKey: any;
966
        if (subproject.enforce_key) {
967
968
            datasetOUT = await DatasetDAO.getByKey(journalClient, datasetIN);
            datasetOUTKey = journalClient.createKey({
969
970
971
972
                namespace: Config.SEISMIC_STORE_NS + '-' + datasetIN.tenant + '-' + datasetIN.subproject,
                path: [Config.DATASETS_KIND],
                enforcedKey: datasetIN.path.slice(0, -1) + '/' + datasetIN.name
            });
973
        } else {
Diego Molteni's avatar
Diego Molteni committed
974
            const results = await DatasetDAO.get(journalClient, datasetIN);
975
976
977
            datasetOUT = results[0];
            datasetOUTKey = results[1];
        }
Diego Molteni's avatar
Diego Molteni committed
978

979
980
981
982
983
984
        // Check if the dataset does not exist
        if (!datasetOUT) {
            throw (Error.make(Error.Status.NOT_FOUND,
                'The dataset ' + Config.SDPATHPREFIX + datasetIN.tenant + '/' +
                datasetIN.subproject + datasetIN.path + datasetIN.name + ' does not exist'));
        }
Diego Molteni's avatar
Diego Molteni committed
985

986
987
988
989
990
        if (datasetOUT.gtags) {
            if (typeof datasetOUT.gtags === 'string') {
                const originalGtags = datasetOUT.gtags;
                datasetOUT.gtags = [];
                datasetOUT.gtags.push(originalGtags);
Diego Molteni's avatar
Diego Molteni committed
991
            }
992
993
994
995
            if (typeof datasetIN