dynamodb.ts 15.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import { TenantModel } from '../../../services/tenant';
16
import { AbstractJournal, AbstractJournalTransaction, IJournalQueryModel, IJournalTransaction, JournalFactory } from '../../journal';
17
18
import { AWSConfig } from './config';
import AWS from 'aws-sdk/global';
19
import DynamoDB, { ScanInput } from 'aws-sdk/clients/dynamodb';
20
21

import aws from 'aws-sdk';
22
import { PromiseResult } from 'aws-sdk/lib/request';
23
import { Utils } from '../../../shared/utils';
24
25
26
27
28
const converter = aws.DynamoDB.Converter;

@JournalFactory.register('aws')
export class AWSDynamoDbDAO extends AbstractJournal {

29
    public KEY = Symbol('id');
30
31
    private dataPartition: string; //tenant name
    private tenant: TenantModel;
32
33
    public constructor(tenant: TenantModel) {
        super();
34
35
        this.tenant = tenant;
        this.dataPartition = tenant.esd.indexOf('.') !== -1 ? tenant.esd.split('.')[0] : tenant.esd;
36
37
38
39
40
41
42
43
        AWS.config.update({ region: AWSConfig.AWS_REGION });
    }

    public async save(datasetEntity: any): Promise<void> {
        if (!(datasetEntity instanceof Array)) {
            datasetEntity = [datasetEntity];
        }
        for (const entity of datasetEntity) {
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
            const item  = entity.data;
            //id attribute is used by aws as partitionKey. 
            //For tenant, id = name, for subproject, id=tenant:name; for dataset, id=tenant:subproject:name:path; for app, id=tenant:email
            var strs = entity.key.partitionKey.split(':');
            if(entity.key.table_kind == AWSConfig.DATASETS_KIND && strs.length == 2){
                //fill in data name and path to the key
                entity.key.partitionKey = entity.key.partitionKey+':'+item.name+':'+item.path;
            }
            if(entity.key.table_kind == AWSConfig.APPS_KIND){
                item['tenant'] = strs[0]; //add tenant entry for App table
            }
            item['id'] = entity.key.partitionKey;
            if (entity.ctag) {
                item['ctag'] = entity.ctag;
            }
            //save extra info as this property will be consumed by the service to identify a data record
            item[this.KEY.toString()] = entity.key;

62
            const itemMarshall = converter.marshall(item);
63
            console.log('from table ' + entity.key.table_name + ' save ' + JSON.stringify(itemMarshall));
64
            const para = {
65
                TableName: entity.key.table_name,
66
67
68
69
70
71
72
73
                Item: itemMarshall
            };
            const db = new DynamoDB({});
            await db.putItem(para).promise();
        }
    }

    public async get(key: any): Promise<[any | any[]]> {
74
75

        const item = { 'id': key.partitionKey };
76
        const itemMarshall = converter.marshall(item);
77
        console.log('from table ' + key.table_name + ' get ' + JSON.stringify(itemMarshall));
78
        const params = {
79
            TableName: key.table_name,
80
81
82
83
84
85
86
            Key: itemMarshall
        };
        const db = new DynamoDB({});
        const data = await db.getItem(params).promise();
        const ret = converter.unmarshall(data.Item);
        if (Object.keys(ret).length === 0)
            return [undefined];
87
88
89
        else {
            //remove aws specific attribute id
            delete ret['id'];
90
91
            // to pass integration test
            delete ret[this.KEY.toString()];
92
            return [ret];
93
        }
94
95
96
    }

    public async delete(key: any): Promise<void> {
97
        const item = { 'id': key.partitionKey };
98
        const itemMarshall = converter.marshall(item);
99
        console.log('from table ' + key.table_name + ' delete ' + JSON.stringify(itemMarshall));
100
        const params = {
101
            TableName: key.table_name,
102
103
104
105
106
107
108
            Key: itemMarshall
        };
        const db = new DynamoDB({});
        await db.deleteItem(params).promise();
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
109
        return new AWSDynamoDbQuery(namespace, kind);
110
111
112
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
113
        const dbQuery = (query as AWSDynamoDbQuery);
114
115
        const statement = dbQuery.getQueryStatement(dbQuery.kind);

116
117
118
119
120
121
122
        console.log('query ' + JSON.stringify(statement));
        const db = new DynamoDB.DocumentClient();
        var scanResults = [];
        var items: PromiseResult<DynamoDB.DocumentClient.ScanOutput, AWS.AWSError>;
        do {
            items = await db.scan(statement).promise();
            const results = items.Items.map(result => {
123
124
125
126
127
                var ret = {};
                ret = result;
                //update object property for service (dao.ts) to consume
                if (ret[this.KEY.toString()]) {
                    ret[this.KEY] = result[this.KEY.toString()];
128
                }
129
                return ret;
130
131
132
133
            });
            scanResults = scanResults.concat(results);
            statement.ExclusiveStartKey = items.LastEvaluatedKey;
        } while (typeof items.LastEvaluatedKey !== "undefined");
134
        return Promise.resolve([scanResults, { endCursor: items.LastEvaluatedKey }]);
135
    }
136

137
    public createKey(specs: any): object {
138
139
140
141
142
143
144
145
146
147
148
149
150
151
        const table_kind = specs.path[0];
        const name = specs.path[1];  //our key
        var partitionKey = name; // partitionKey

        var strs = specs.namespace.split('-');
        if (table_kind === AWSConfig.SUBPROJECTS_KIND) {
            partitionKey = strs[strs.length - 1] + ':' + partitionKey; //tenant:subprojet for id
        }
        if (table_kind === AWSConfig.DATASETS_KIND) {
            partitionKey = strs[strs.length - 2] + ':' + strs[strs.length - 1]; //tenant:subprojet for id
        }
        if (table_kind === AWSConfig.APPS_KIND) {
            partitionKey = strs[strs.length - 1] + ':' + name; //tenant:subprojet for id
        }
Yunhua Koglin's avatar
Yunhua Koglin committed
152
153

        const table_name = AWSConfig.AWS_ENVIRONMENT + '-' + 'SeismicStore.' + specs.path[0];
154
        return { table_name, name, table_kind, partitionKey };
155
156
157
    }

    public getTransaction(): IJournalTransaction {
158
        return new AWSDynamoDbTransactionDAO(this);
159
160
161
    }

    public getQueryFilterSymbolContains(): string {
162
        return 'CONTAINS';
163
164
165
    }
}

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261

declare type OperationType = 'save' | 'delete';
export class AWSDynamoDbTransactionOperation {

    public constructor(type: OperationType, entityOrKey: any) {
        this.type = type;
        this.entityOrKey = entityOrKey;
    }

    public type: OperationType;
    public entityOrKey: any;
}

export class AWSDynamoDbTransactionDAO extends AbstractJournalTransaction {

    public KEY = null;

    public constructor(owner: AWSDynamoDbDAO) {
        super();
        this.owner = owner;
        this.KEY = this.owner.KEY;
    }

    public async save(entity: any): Promise<void> {
        console.log('aws Transaction Save ' + JSON.stringify(entity));
        this.queuedOperations.push(new AWSDynamoDbTransactionOperation('save', entity));
        await Promise.resolve();
    }

    public async get(key: any): Promise<[any | any[]]> {
        console.log('aws Transaction get ' + JSON.stringify(key));
        return await this.owner.get(key);
    }

    public async delete(key: any): Promise<void> {
        console.log('aws Transaction delete ' + JSON.stringify(key));
        this.queuedOperations.push(new AWSDynamoDbTransactionOperation('delete', key));
        await Promise.resolve();
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
        console.log('aws Transaction createQuery ' + namespace + kind);
        return this.owner.createQuery(namespace, kind);
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
        console.log('aws Transaction runQuery ' + JSON.stringify(query));
        return await this.owner.runQuery(query);
    }

    public async run(): Promise<void> {
        console.log('aws Transaction run ');
        if (this.queuedOperations.length) {
            await Promise.reject('Transaction is already in use.');
        }
        else {
            this.queuedOperations = [];
            return Promise.resolve();
        }
    }

    public async rollback(): Promise<void> {
        console.log('aws Transaction rollback ');
        this.queuedOperations = [];
        return Promise.resolve();
    }

    public async commit(): Promise<void> {
        console.log('aws Transaction commit ');
        for (const operation of this.queuedOperations) {
            if (operation.type === 'save') {
                await this.owner.save(operation.entityOrKey);
            }
            if (operation.type === 'delete') {
                await this.owner.delete(operation.entityOrKey);
            }
        }

        this.queuedOperations = [];
        return Promise.resolve();
    }

    public getQueryFilterSymbolContains(): string {
        return 'contains';
    }

    private owner: AWSDynamoDbDAO;
    public queuedOperations: AWSDynamoDbTransactionOperation[] = [];
}

declare type Operator = '=' | '<' | '>' | '<=' | '>=' | 'HAS_ANCESTOR' | 'CONTAINS';
export class AWSDynamoDbQuery implements IJournalQueryModel {

    public constructor(namespace: string, kind: string) {
        this.namespace = namespace;
        this.kind = kind;
Yunhua Koglin's avatar
Yunhua Koglin committed
262
        this.queryStatement = { TableName: kind, FilterExpression: '', ExpressionAttributeNames: {}, ExpressionAttributeValues: {}, ProjectionExpression:''  };
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
    }
    public namespace: string;
    public kind: string;
    public queryStatement: ScanInput;

    filter(property: string, value: {}): IJournalQueryModel;

    filter(property: string, operator: Operator, value: {}): IJournalQueryModel;

    filter(property: string, operator?: Operator, value?: {}): IJournalQueryModel {
        if (operator === 'CONTAINS') {
            this.queryStatement.FilterExpression += 'contains(#' + property + ',:' + property + ')';
            this.queryStatement.ExpressionAttributeNames['#' + property] = property;
            this.queryStatement.ExpressionAttributeValues[':' + property] = value;
            return this;
        }
        if (value === undefined) {
            value = operator;
            operator = '=';
        }
        if (operator === undefined) {
            operator = '=';
        }
        if (value === undefined) {
            value = '';
        }

        if (operator === 'HAS_ANCESTOR') {
            throw new Error('HAS_ANCESTOR operator is not supported in query filters.');
        }
293
294

        if (!!(this.queryStatement.FilterExpression)) {
295
            this.queryStatement.FilterExpression += ' AND ';
296
        }
297

Yunhua Koglin's avatar
Yunhua Koglin committed
298
299
300
301
302
303
304
305
306
307
308
309
310
        if(property ==='path')
        {
            property='p';
            const pathproperty='path';
            this.queryStatement.FilterExpression += '#' + property + operator + ':' + property;
            this.queryStatement.ExpressionAttributeNames['#' + property] = pathproperty;
            this.queryStatement.ExpressionAttributeValues[':' + property] = value;
        }
        else {
            this.queryStatement.FilterExpression += '#' + property + operator + ':' + property;
            this.queryStatement.ExpressionAttributeNames['#' + property] = property;
            this.queryStatement.ExpressionAttributeValues[':' + property] = value;
        }
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        return this;
    }

    start(start: string | Buffer): IJournalQueryModel {
        if (start instanceof Buffer) {
            throw new Error('Type \'Buffer\' is not supported for DynamoDB Continuation while paging.');
        }
        console.log('NOT SUPPOR aws start createQuery ' + start);
        return this;
    }

    limit(n: number): IJournalQueryModel {
        this.queryStatement.Limit = n;
        return this;
    }

    groupBy(fieldNames: string | string[]): IJournalQueryModel {
        console.log('NOT SUPPORT aws groupBy createQuery ' + fieldNames);
        return this;
    }

    select(fieldNames: string | string[]): IJournalQueryModel {
        if (this.queryStatement.ProjectionExpression.length >= 1)
            this.queryStatement.ProjectionExpression += ',';
        if (typeof fieldNames === 'string') {
            this.queryStatement.ProjectionExpression += fieldNames;
        } else {
Yunhua Koglin's avatar
Yunhua Koglin committed
338
339
340
            if(fieldNames[0] === 'path')
            this.queryStatement.ProjectionExpression += '#p';
            else
341
342
343
344
345
            this.queryStatement.ProjectionExpression += fieldNames.join(',');
        }
        return this;
    }
    public getQueryStatement(tableName: string): ScanInput {
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377

        //since we have one table for all datasets, we need to add more filters to return dataset specific for that tenant/subproject
        var strs = this.namespace.split('-');
        if (this.kind === AWSConfig.DATASETS_KIND || this.kind === AWSConfig.SUBPROJECTS_KIND) { //one table, filter on tenant
            if (!!(this.queryStatement.FilterExpression)) {
                this.queryStatement.FilterExpression += ' AND ';
            }
            const t_property: string = 'tenant';
            var value = {}; value = this.kind === AWSConfig.DATASETS_KIND ? strs[strs.length - 2] : strs[strs.length - 1];
            this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
            this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
            this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
        }
        if (this.kind === AWSConfig.DATASETS_KIND) { // one table, filter on subproject too
            this.queryStatement.FilterExpression += ' AND ';
            const t_property: string = 'subproject';
            var value = {}; value = strs[strs.length - 1];
            this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
            this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
            this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
        }

        if (this.kind === AWSConfig.APPS_KIND) { // one table, filter on tenant
            if (!!(this.queryStatement.FilterExpression)) {
                this.queryStatement.FilterExpression += ' AND ';
            }
            const t_property: string = 'tenant';
            var value = {}; value = strs[strs.length - 1];
            this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
            this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
            this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
        }
Yunhua Koglin's avatar
Yunhua Koglin committed
378

379
380
        if (this.queryStatement.FilterExpression.length === 0)
            delete this.queryStatement.FilterExpression;
Yunhua Koglin's avatar
Yunhua Koglin committed
381
382
383
        
        if (this.queryStatement.ProjectionExpression.length === 0)
            delete this.queryStatement.ProjectionExpression;
384

385
386
387
388
389
        //delete empty objects in query parameters
        if (Object.entries(this.queryStatement.ExpressionAttributeNames).length === 0) {
            delete this.queryStatement.ExpressionAttributeNames;
            delete this.queryStatement.ExpressionAttributeValues;
        }
390

Yunhua Koglin's avatar
Yunhua Koglin committed
391
        this.queryStatement.TableName = AWSConfig.AWS_ENVIRONMENT + '-SeismicStore.' + tableName;
392
393
394
        return this.queryStatement;
    }
}