datastore.ts 19.5 KB
Newer Older
1
2
/* Licensed Materials - Property of IBM              */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
3
4

import { AbstractJournal, AbstractJournalTransaction, IJournalQueryModel, IJournalTransaction, JournalFactory } from '../../journal';
5
import { TenantModel } from '../../../services/tenant';
6
import cloudant from '@cloudant/cloudant';
7
8
9
10
11
12
13
14
import { Config } from '../../config';
import { Utils } from '../../../shared/utils'
import { IbmConfig } from './config';
import { logger } from './logger';

@JournalFactory.register('ibm')
export class DatastoreDAO extends AbstractJournal {
    public KEY = Symbol('id');
15
    private dataPartition: string;
16
    private docDb;
17

18
    public constructor(tenant: TenantModel) {
19
20
        super();
        logger.info('In datastore.constructor.');
Walter D's avatar
Walter D committed
21
        this.dataPartition = tenant.gcpid;
22
23
24
25
26
    }

    public async initDb(dataPartition: string)
    {
        logger.info('In datastore.initDb.');
27
        const dbUrl = IbmConfig.DOC_DB_URL;
28
        const cloudantOb = cloudant(dbUrl);
29
        logger.info('DB initialized. cloudantOb-');
30

31
        try {
32
33
34
            logger.debug('Before DB connection');
            this.docDb = await cloudantOb.db.get(IbmConfig.DOC_DB_COLLECTION + '-' + dataPartition);
            logger.debug('Got DB connection');
35
        } catch (err) {
36
37
38
39
40
41
            if(err.statusCode === 404)
            {
                logger.debug('Database does not exist. Creating database.');
                await cloudantOb.db.create(IbmConfig.DOC_DB_COLLECTION + '-' + dataPartition)
                logger.debug('Database created.');
            }
42
43
44
45
46
            else
            {
                logger.debug('DB connection error - ', err);
                return;
            }
47
        }
48

49
        this.docDb = cloudantOb.db.use(IbmConfig.DOC_DB_COLLECTION + '-' + dataPartition);
50

51
    }
52
53
54

    public async get(key: any): Promise<[any | any[]]> {
        logger.info('In datastore get.');
Walter D's avatar
Walter D committed
55
        logger.debug(key);
56
        let entityDocument;
57
        await this.initDb(this.dataPartition);
58
        /// using the field 'name' to fetch the document. Note: the get() is expecting the field _id
59
        entityDocument = await this.docDb.get(key.name).then(
60
61
62
            result => {
                result[this.KEY] = result[this.KEY.toString()];
                delete result[this.KEY.toString()];
Walter D's avatar
Walter D committed
63
64
                logger.info('Deleted field');
                logger.debug(result[this.KEY.toString()]);
65
66
67
                return [result];
            }
        ).catch((error) => {
Walter D's avatar
Walter D committed
68
69
            logger.error('Get failed to fetch the document.');
            logger.error(error);
70
            return [undefined];
71
        })
Walter D's avatar
Walter D committed
72
        logger.debug(entityDocument);
73
        logger.info('returning from datastore');
74
        return entityDocument;
75
    }
76
77


78
79
    public async save(entity: any): Promise<void> {
        logger.info('In datastore.save.');
Walter D's avatar
Walter D committed
80
        logger.debug(entity);
81
        const self = this;
82
83
        logger.info('Connecting to DB.');
        await this.initDb(this.dataPartition);
84
        logger.info('Fetching document.');
85

Walter D's avatar
Walter D committed
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
        try{
            const getResponse = await this.docDb.get(entity.key.name, { revs_info: true });
            logger.info('Document exists in db.');
            const existingDoc = getResponse;
            const docTemp = JSON.parse(JSON.stringify(existingDoc));
            /// have to add if condition. before that check the dataset object structure
            docTemp.ltag = entity.ltag;
            if (entity.data.trusted)
                docTemp.trusted = entity.data.trusted;

            Object.assign(docTemp, entity.data);
            logger.debug(docTemp);
            await this.docDb.insert(docTemp, entity.key.name);
            logger.info('Document updated.');
        } catch(err){
            if(err.statusCode === 404)
102
103
            {
                logger.info('Document does not exist. This will be a new document');
104
                const customizedOb = {};
105
106
107
                customizedOb['id'] = entity.key.name;
                customizedOb['key'] = entity.key.partitionKey;
                customizedOb[self.KEY.toString()] = entity.key;
108
109
110
                for (const element in entity.data) {
                    if (!((entity.key.kind === 'datasets' || entity.key.kind === 'seismicmeta') && element === '_id'))
                        if (!((entity.key.kind === 'datasets' || entity.key.kind === 'seismicmeta') && element === '_rev'))
Walter D's avatar
Walter D committed
111
                                    customizedOb[element] = entity.data[element];
112
                };
Walter D's avatar
Walter D committed
113
                logger.debug(customizedOb);
Walter D's avatar
Walter D committed
114
                await this.docDb.insert(customizedOb, entity.key.name);
115
116
                logger.info('Document inserted.');
            }
Walter D's avatar
Walter D committed
117
        }
118
        logger.info('Returning from datastore.save.');
119
120
121

    }

122
123
    public async delete(key: any): Promise<void> {
        logger.info('In datastore.delete.');
124
125
126
        logger.info('Connecting to DB.');
        await this.initDb(this.dataPartition);
        const doc = await this.docDb.get(key.name);
127
        try {
128
            this.docDb.destroy(doc._id, doc._rev);
129
130
            logger.info('Document deleted.');
        }
131
        catch (err) {
Walter D's avatar
Walter D committed
132
133
            logger.error('Deletion failed. Error - ');
            logger.error(err);
134
135
        }
        logger.info('Returning from datastore.delete.');
136
137
138
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
139
        logger.info('In datastore.createQuery. Returning.');
Walter D's avatar
Walter D committed
140
141
        logger.debug(namespace);
        logger.debug(kind);
142
143
144
145
        return new IbmDocDbQuery(namespace, kind);
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
146
147
        logger.info('In datastore.runQuery.');
        const queryObject = (query as IbmDocDbQuery);
Walter D's avatar
Walter D committed
148
        logger.debug(queryObject);
149
150
        /// tablemane datasets??
        const mangoQuery = queryObject.prepareStatement(Config.DATASETS_KIND, queryObject.namespace, queryObject.kind);
Walter D's avatar
Walter D committed
151
        logger.debug(mangoQuery);
152
153

        let docs;
154
155
156
        logger.info('Connecting to DB.');
        await this.initDb(this.dataPartition);
        await this.docDb.find(mangoQuery).then((doc) => {
157
            docs = doc.docs;
158
159
            logger.debug(doc.docs);
        });
Walter D's avatar
Walter D committed
160
        logger.info('Find query executed.');
161

162
163
164
165
166
167
168
169
170
171
172
173
        const results = docs.map(result => {
            if (!result) {
                return result;
            } else {
                if (result[this.KEY.toString()]) {
                    result[this.KEY] = result[this.KEY.toString()];
                    delete result[this.KEY.toString()];
                    return result;
                } else {
                    return result;
                }
            }
174
        });
175
        return Promise.resolve([results, {}]);
176
177
178
    }

    public createKey(specs: any): object {
179
        logger.info('In datastore.createKey');
Walter D's avatar
Walter D committed
180
        logger.debug(specs);
181
182
183
184
185
186
        const kind = specs.path[0];
        const partitionKey = specs.namespace + '-' + kind;
        let name: string;
        if (kind === Config.DATASETS_KIND) {
            name = Utils.makeID(16);
        } else if (kind === Config.SEISMICMETA_KIND) {
187
            name = specs.path[1].replace(/\W/g, '-');/// replaces path slashes into hyphen
188
189
190
        } else {
            name = specs.path[1];
        }
Walter D's avatar
Walter D committed
191
192
193
        logger.debug(name);
        logger.debug(partitionKey);
        logger.debug(kind);
194
195
        logger.info('returning from createKey');
        return { name, partitionKey, kind };
196
197
198
    }

    public getTransaction(): IJournalTransaction {
199
200
201
        logger.info('In datastore.getTransaction');
        return new IbmDocDbTransactionDAO(this);
    }
202

203
204
    public getQueryFilterSymbolContains(): string {
        logger.info('In datastore.getQueryFilterSymbolContains. Not implemented');
205
        return 'CONTAINS';// not implemented
206
207
208
209
210
211
    }
}

declare type OperationType = 'save' | 'delete';
export class IbmDocDbTransactionOperation {

212
    public constructor(type: OperationType, entityOrKey: any) {
213
        logger.info('In datastore.IbmDocDbTransactionOperation.constructor.');
Walter D's avatar
Walter D committed
214
215
        logger.debug(type);
        logger.debug(entityOrKey);
216
217
218
219
220
221
222
223
224
225
        this.type = type;
        this.entityOrKey = entityOrKey;
    }

    public type: OperationType;
    public entityOrKey: any;
}

/**
 * A wrapper class for datastore transactions
226
 * ! Note: looks awfully close to datastore interface.
227
228
229
230
231
232
233
234
 */
export class IbmDocDbTransactionDAO extends AbstractJournalTransaction {

    public KEY = null;

    public constructor(owner: DatastoreDAO) {
        super();
        logger.info('In datastore.IbmDocDbTransactionDAO.constructor.');
Walter D's avatar
Walter D committed
235
        logger.debug(owner);
236
237
238
239
240
241
        this.owner = owner;
        this.KEY = this.owner.KEY;
    }

    public async save(entity: any): Promise<void> {
        logger.info('In datastore.IbmDocDbTransactionDAO.save.');
Walter D's avatar
Walter D committed
242
        logger.debug(entity);
243
244
245
246
247
248
        this.queuedOperations.push(new IbmDocDbTransactionOperation('save', entity));
        await Promise.resolve();
    }

    public async get(key: any): Promise<[any | any[]]> {
        logger.info('In datastore.IbmDocDbTransactionDAO.get.');
Walter D's avatar
Walter D committed
249
        logger.debug(key);
250
251
252
253
254
        return await this.owner.get(key);
    }

    public async delete(key: any): Promise<void> {
        logger.info('In datastore.IbmDocDbTransactionDAO.delete.');
Walter D's avatar
Walter D committed
255
        logger.debug(key);
256
257
258
259
260
261
        this.queuedOperations.push(new IbmDocDbTransactionOperation('delete', key));
        await Promise.resolve();
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
        logger.info('In datastore.IbmDocDbTransactionDAO.createQuery.');
Walter D's avatar
Walter D committed
262
263
        logger.debug(namespace);
        logger.debug(kind);
264
265
266
267
268
        return this.owner.createQuery(namespace, kind);
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
        logger.info('In datastore.IbmDocDbTransactionDAO.runQuery.');
Walter D's avatar
Walter D committed
269
        logger.debug(query)
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
        return await this.owner.runQuery(query);
    }

    public async run(): Promise<void> {
        logger.info('In datastore.IbmDocDbTransactionDAO.run.');
        if (this.queuedOperations.length) {
            await Promise.reject('Transaction is already in use.');
        }
        else {
            this.queuedOperations = [];
            return Promise.resolve();
        }
    }

    public async rollback(): Promise<void> {
        logger.info('In datastore.IbmDocDbTransactionDAO.rollback.');
        this.queuedOperations = [];
        return Promise.resolve();
    }

    public async commit(): Promise<void> {
        logger.info('In datastore.IbmDocDbTransactionDAO.commit.');
292
        for (const operation of this.queuedOperations) {
293
294
295
296
297
298
299
300
301
302
303
304
            if (operation.type === 'save') {
                await this.owner.save(operation.entityOrKey);
            }
            if (operation.type === 'delete') {
                await this.owner.delete(operation.entityOrKey);
            }
        }

        this.queuedOperations = [];
        return Promise.resolve();
    }

305
    public getQueryFilterSymbolContains(): string {
306
307
308
309
310
311
312
313
314
315
        logger.info('In datastore.IbmDocDbTransactionDAO.getQueryFilterSymbolContains. Not implemented');
        return '';
    }

    private owner: DatastoreDAO;
    public queuedOperations: IbmDocDbTransactionOperation[] = [];
}


/**
316
 * not sure of HAS_ANCESTOR and CONTAINS in CouchDB.
317
318
319
320
321
322
 */
declare type Operator = '=' | '<' | '>' | '<=' | '>=' | 'HAS_ANCESTOR' | 'CONTAINS';

/*
declaring enum for operator
*/
323
324
325
326
327
328
329
enum CouchOperators {
    Equal = '$eq',
    GreaterThan = '$gt',
    LesserThan = '$lt',
    GreaterThanEqualTo = '$gte',
    LesserThanEqualTo = '$lte',
    Contains = '$elemMatch',
330
331
}

332
333
334
335
336
337
338
enum ConditionalOperators {
    Equal = '=',
    GreaterThan = '>',
    LesserThan = '<',
    GreaterThanEqualTo = '>=',
    LesserThanEqualTo = '<=',
    Contains = 'CONTAINS',
339
340
341
342
343
344
345
346
}


/**
 * implementation of IJournalQueryModel
 */
export class IbmDocDbQuery implements IJournalQueryModel {

347
    public namespace: string;
348
349
350
351
    public kind: string;

    public constructor(namespace: string, kind: string) {
        logger.info('In datastore.IbmDocDbQuery.constructor.');
Walter D's avatar
Walter D committed
352
353
        logger.debug(namespace);
        logger.debug(kind);
354
355
356
357
        this.namespace = namespace;
        this.kind = kind;
    }

358

359
360
361
362
363
364
365
366

    filter(property: string, value: {}): IJournalQueryModel;

    filter(property: string, operator: Operator, value: {}): IJournalQueryModel;

    filter(property: string, operator?: Operator, value?: {}): IJournalQueryModel {
        logger.info('In datastore.IbmDocDbQuery.filter.');
        logger.info('in filter("esd","=",esd)');
Walter D's avatar
Walter D committed
367
368
369
370
        logger.debug(property);
        logger.debug(operator);
        logger.debug(value);

371
372
373
374
375
376
377
378
379
380
381
        if (value === undefined) {
            value = operator;
            operator = '=';
        }
        if (operator === undefined) {
            operator = '=';
        }
        if (value === undefined) {
            value = '';
        }

382
        logger.info('modified values');
Walter D's avatar
Walter D committed
383
384
385
        logger.debug(property);
        logger.debug(operator);
        logger.debug(value);
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404

        let cdbOperator;

        switch (operator) {
            case ConditionalOperators.Equal:
                cdbOperator = CouchOperators.Equal;
                break;
            case ConditionalOperators.GreaterThan:
                cdbOperator = CouchOperators.GreaterThan
                break;
            case ConditionalOperators.GreaterThanEqualTo:
                cdbOperator = CouchOperators.GreaterThanEqualTo
                break;
            case ConditionalOperators.LesserThan:
                cdbOperator = CouchOperators.LesserThan
                break;
            case ConditionalOperators.LesserThanEqualTo:
                cdbOperator = CouchOperators.LesserThanEqualTo
                break;
405
406
407
            case ConditionalOperators.Contains:
                cdbOperator = CouchOperators.Contains
                break;
408
        }
409

410
        logger.debug('cdbOperator - ' + cdbOperator);
411
412
413

        const filter = new IbmQueryFilter(property, cdbOperator, value);
        this.filters.push(filter);
Walter D's avatar
Walter D committed
414
        logger.debug(filter);
415
416
417
418
419
420
421
422
423
424
425
        return this;
    }

    start(start: string | Buffer): IJournalQueryModel {
        logger.info('In datastore.IbmDocDbQuery.start. Have to work on this.');
        return this;
    }

    limit(n: number): IJournalQueryModel {
        logger.info('In datastore.IbmDocDbQuery.limit.');
        this.pagingLimit = n;
Walter D's avatar
Walter D committed
426
        logger.debug(this.pagingLimit);
427
428
429
430
431
        return this;
    }

    groupBy(fieldNames: string | string[]): IJournalQueryModel {
        logger.info('In datastore.IbmDocDbQuery.groupBy. Have to work on this.');
Walter D's avatar
Walter D committed
432
        logger.debug(fieldNames);
433
434
435
        return this;
    }

436
    /// field names added to query. Returned in the response if they exists.
437
    select(fieldNames: string | string[]): IJournalQueryModel {
438
        /// if you wondering, converts string to an array
439
        logger.info('In datastore.IbmDocDbQuery.select.');
Walter D's avatar
Walter D committed
440
        logger.debug(fieldNames);
441
        if (typeof fieldNames === 'string') {
442
443
444
445
446
447
448
449
450
451
452
            this.projectedFieldNames = [fieldNames];
        } else {
            this.projectedFieldNames = fieldNames;
        }
        return this;
    }

    private filters: IbmQueryFilter[] = [];
    private projectedFieldNames: string[] = [];
    private groupByFieldNames: string[] = [];
    private pagingStart?: string;
453
454
455
    private pagingLimit?: number;

    // public prepareSqlStatement(tableName: string): { spec: SqlQuerySpec, options: FeedOptions } {
456
457
    public prepareStatement(tableName: string, namespace: string, kind: string): any {
        logger.info('In datastore.IbmDocDbQuery.prepareStatement.');
Walter D's avatar
Walter D committed
458
459
460
        logger.debug(tableName);
        logger.debug(namespace);
        logger.debug(kind);
461
        const builder = new QueryStatementBuilder(tableName, namespace, kind);
Walter D's avatar
Walter D committed
462
        logger.debug(builder);
463

464
        for (const filter of this.filters) {
465
466
467
468
469
            filter.addFilterExpression(builder);
        }

        builder.projectedFieldNames = this.projectedFieldNames;
        builder.pagingLimit = this.pagingLimit;
470
        /*builder.groupByFieldNames = this.groupByFieldNames;*//// working on basic query
471
472

        const spec = builder.build();
Walter D's avatar
Walter D committed
473
        logger.debug(spec);
474
475
476
477
478
479
480
481
482
483
484
485
486
        return spec;
    }

}


class IbmQueryFilter {

    public constructor(property: string, operator: string, value: {}) {
        logger.info('In datastore.IbmQueryFilter.constructor.');
        this.property = property;
        this.operator = operator;
        this.value = value;
Walter D's avatar
Walter D committed
487
488
489
        logger.debug(this.property);
        logger.debug(this.operator);
        logger.debug(this.value);
490
491
492
493
494
495
496
497
498
499
    }

    public property: string;

    public operator: string;

    public value: {};

    public addFilterExpression(toStatement: QueryStatementBuilder) {
        logger.info('In datastore.IbmQueryFilter.addFilterExpression.');
Walter D's avatar
Walter D committed
500
        logger.debug(toStatement);
501
502
503
504
505
        toStatement.addFilterExpression(this.property, this.operator, this.value);
    }
}

class QueryStatementBuilder {
506
    public tableName: string;
507
508
509
510
511
512
513
514
515
516
517
    public namespace: string;
    public kind: string;
    private filterExpressions: string[] = [];
    public projectedFieldNames: string[] = [];
    public pagingLimit?: number;

    constructor(tableName: string, namespace: string, kind: string) {
        logger.info('In datastore.QueryStatementBuilder.constructor.');
        this.tableName = tableName;
        this.namespace = namespace;
        this.kind = kind;
Walter D's avatar
Walter D committed
518
519
520
        logger.debug(this.tableName);
        logger.debug(this.namespace);
        logger.debug(this.kind);
521
    }
522

523
524
    public addFilterExpression(property: string, operator: string, value: {}) {
        logger.info('In datastore.QueryStatementBuilder.addFilterExpression.');
Walter D's avatar
Walter D committed
525
526
527
        logger.debug(property);
        logger.debug(operator);
        logger.debug(value);
528
529
530
531
        this.filterExpressions.push(
            '{"property": "' + property
            + '", "operator": "' + operator
            + '", "value": "' + value + '"}');
532
533
    }

534
535
    // public build(): SqlQuerySpec {
    public build(): any {
536
537

        logger.info('In datastore.QueryStatementBuilder.build');
538
539
        const selectorQuery = {};
        const andWrapper = {};
540
541
        andWrapper['$and'] = [];

542
543
544
        const keyQuery = {};
        /// let filter = '';
        const fieldsOption = [];
545

546
        keyQuery['Symbol(id)'] = { partitionKey: { $eq: this.namespace + '-' + this.kind }, kind: { $eq: this.kind } };
547
548
549
        andWrapper['$and'].push(keyQuery);
        selectorQuery['selector'] = andWrapper;

550
551
552
        for (const filter of this.filterExpressions) {
            const filterObject = JSON.parse(filter);
            const op = filterObject.operator;
553
554

            let filterQuery = {};
555
            if (filterObject.operator === '$elemMatch') {
556
                logger.debug('$elemMatch operator');
557
                filterQuery = { [filterObject.property]: { [filterObject.operator]: { '$eq': filterObject.value } } };
558
559
            }
            else
560
                filterQuery = { [filterObject.property]: { [filterObject.operator]: filterObject.value } };
561
562
            logger.debug('filterQuery - ');
            logger.debug(filterQuery);
563
564
565
            andWrapper['$and'].push(filterQuery);
        }

Walter D's avatar
Walter D committed
566
567
        logger.debug('Created query AND clause - ');
        logger.debug(andWrapper);
568
569
570

        if (this.projectedFieldNames.length) {
            selectorQuery[IbmConfig.DOC_DB_QUERY_SELECT_FIELDS] = fieldsOption;
571
            for (const field of this.projectedFieldNames) {
572
573
574
575
                fieldsOption.push(field);
            }
        }

576
        if (this.pagingLimit === undefined)
577
            this.pagingLimit = IbmConfig.DOC_DB_QUERY_RESULT_LIMIT_VALUE;
578

579
        selectorQuery[IbmConfig.DOC_DB_QUERY_RESULT_LIMIT] = this.pagingLimit;
Walter D's avatar
Walter D committed
580
        logger.debug(selectorQuery);
581
        return selectorQuery;
582
    }
Walter D's avatar
Walter D committed
583
}