dynamodb.ts 11.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import { TenantModel } from '../../../services/tenant';
16
import { AbstractJournal, AbstractJournalTransaction, IJournalQueryModel, IJournalTransaction, JournalFactory } from '../../journal';
17
18
import { AWSConfig } from './config';
import AWS from 'aws-sdk/global';
19
import DynamoDB, { ScanInput } from 'aws-sdk/clients/dynamodb';
20
21

import aws from 'aws-sdk';
22
import { PromiseResult } from 'aws-sdk/lib/request';
23
24
25
26
27
const converter = aws.DynamoDB.Converter;

@JournalFactory.register('aws')
export class AWSDynamoDbDAO extends AbstractJournal {

28
    public KEY = Symbol('id');
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
    public constructor(tenant: TenantModel) {
        super();
        AWS.config.update({ region: AWSConfig.AWS_REGION });

    }

    public async save(datasetEntity: any): Promise<void> {
        if (!(datasetEntity instanceof Array)) {
            datasetEntity = [datasetEntity];
        }
        for (const entity of datasetEntity) {
            const item = entity.data;
            const itemMarshall = converter.marshall(item);
            console.log('from table ' + entity.key.kind + ' save ' + JSON.stringify(itemMarshall));
            const para = {
                TableName: entity.key.kind,
                Item: itemMarshall
            };
            const db = new DynamoDB({});
            await db.putItem(para).promise();
        }
    }

    public async get(key: any): Promise<[any | any[]]> {
        const item = { 'name': key.partitionKey };
        const itemMarshall = converter.marshall(item);
        console.log('from table ' + key.kind + ' get ' + JSON.stringify(itemMarshall));
        const params = {
            TableName: key.kind,
            Key: itemMarshall
        };
        const db = new DynamoDB({});
        const data = await db.getItem(params).promise();
        const ret = converter.unmarshall(data.Item);
        if (Object.keys(ret).length === 0)
            return [undefined];
        else
            return [ret];
    }

    public async delete(key: any): Promise<void> {
        const item = { 'name': key.partitionKey };
        const itemMarshall = converter.marshall(item);
        console.log('from table ' + key.kind + ' delete ' + JSON.stringify(itemMarshall));
        const params = {
            TableName: key.kind,
            Key: itemMarshall
        };
        const db = new DynamoDB({});
        await db.deleteItem(params).promise();
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
82
        return new AWSDynamoDbQuery(namespace, kind);
83
84
85
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        const dbQuery = (query as AWSDynamoDbQuery);
        const statement = dbQuery.getQueryStatement(AWSConfig.DATASETS_KIND);
        // const statement = {
        //     TableName : 'osdu-kogliny-SeismicStore.datasets',
        //     FilterExpression: "#path=:path",
        //     KeyConditionExpression: '#name=:name',
        //     ExpressionAttributeNames: {
        //          "#name": "name",
        //          "#path": "path"
        //     },
        //     ExpressionAttributeValues: {
        //         ":name": 'yk-2dataset',
        //         ":path": '/a/b/c/'
        //     }       
        // };
        console.log('query ' + JSON.stringify(statement));
        const db = new DynamoDB.DocumentClient();
        var scanResults = [];
        var items: PromiseResult<DynamoDB.DocumentClient.ScanOutput, AWS.AWSError>;
        do {
            items = await db.scan(statement).promise();
            const results = items.Items.map(result => {
                if (!result.data) {
                    return result;
                } else {
                    if (result.data[this.KEY.toString()]) {
                        result.data[this.KEY] = result.data[this.KEY.toString()];
                        delete result.data[this.KEY.toString()];
                        return result.data;
                    } else {
                        return result.data;
                    }
                }
            });
            scanResults = scanResults.concat(results);
            statement.ExclusiveStartKey = items.LastEvaluatedKey;
        } while (typeof items.LastEvaluatedKey !== "undefined");
        return Promise.resolve( [scanResults, {endCursor: items.LastEvaluatedKey}]);
124
    }
125

126
127
128
129
130
131
132
133
134
    public createKey(specs: any): object {
        const kind0 = specs.path[0];
        const partitionKey = specs.path[1];
        const kind = AWSConfig.SERVICE_ENV + '-' + 'SeismicStore.' + kind0;
        //kind is the table name
        return { partitionKey, kind };
    }

    public getTransaction(): IJournalTransaction {
135
        return new AWSDynamoDbTransactionDAO(this);
136
137
138
    }

    public getQueryFilterSymbolContains(): string {
139
        return 'contains';
140
141
142
    }
}

143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

declare type OperationType = 'save' | 'delete';
export class AWSDynamoDbTransactionOperation {

    public constructor(type: OperationType, entityOrKey: any) {
        this.type = type;
        this.entityOrKey = entityOrKey;
    }

    public type: OperationType;
    public entityOrKey: any;
}

export class AWSDynamoDbTransactionDAO extends AbstractJournalTransaction {

    public KEY = null;

    public constructor(owner: AWSDynamoDbDAO) {
        super();
        this.owner = owner;
        this.KEY = this.owner.KEY;
    }

    public async save(entity: any): Promise<void> {
        console.log('aws Transaction Save ' + JSON.stringify(entity));
        this.queuedOperations.push(new AWSDynamoDbTransactionOperation('save', entity));
        await Promise.resolve();
    }

    public async get(key: any): Promise<[any | any[]]> {
        console.log('aws Transaction get ' + JSON.stringify(key));
        return await this.owner.get(key);
    }

    public async delete(key: any): Promise<void> {
        console.log('aws Transaction delete ' + JSON.stringify(key));
        this.queuedOperations.push(new AWSDynamoDbTransactionOperation('delete', key));
        await Promise.resolve();
    }

    public createQuery(namespace: string, kind: string): IJournalQueryModel {
        console.log('aws Transaction createQuery ' + namespace + kind);
        return this.owner.createQuery(namespace, kind);
    }

    public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
        console.log('aws Transaction runQuery ' + JSON.stringify(query));
        return await this.owner.runQuery(query);
    }

    public async run(): Promise<void> {
        console.log('aws Transaction run ');
        if (this.queuedOperations.length) {
            await Promise.reject('Transaction is already in use.');
        }
        else {
            this.queuedOperations = [];
            return Promise.resolve();
        }
    }

    public async rollback(): Promise<void> {
        console.log('aws Transaction rollback ');
        this.queuedOperations = [];
        return Promise.resolve();
    }

    public async commit(): Promise<void> {
        console.log('aws Transaction commit ');
        for (const operation of this.queuedOperations) {
            if (operation.type === 'save') {
                await this.owner.save(operation.entityOrKey);
            }
            if (operation.type === 'delete') {
                await this.owner.delete(operation.entityOrKey);
            }
        }

        this.queuedOperations = [];
        return Promise.resolve();
    }

    public getQueryFilterSymbolContains(): string {
        return 'contains';
    }

    private owner: AWSDynamoDbDAO;
    public queuedOperations: AWSDynamoDbTransactionOperation[] = [];
}

declare type Operator = '=' | '<' | '>' | '<=' | '>=' | 'HAS_ANCESTOR' | 'CONTAINS';
export class AWSDynamoDbQuery implements IJournalQueryModel {

    public constructor(namespace: string, kind: string) {
        this.namespace = namespace;
        this.kind = kind;
        this.queryStatement = { TableName: kind, FilterExpression: '', ExpressionAttributeNames: {}, ExpressionAttributeValues: {} };
    }
    public namespace: string;
    public kind: string;
    public queryStatement: ScanInput;

    filter(property: string, value: {}): IJournalQueryModel;

    filter(property: string, operator: Operator, value: {}): IJournalQueryModel;

    filter(property: string, operator?: Operator, value?: {}): IJournalQueryModel {
        if (operator === 'CONTAINS') {
            this.queryStatement.FilterExpression += 'contains(#' + property + ',:' + property + ')';
            this.queryStatement.ExpressionAttributeNames['#' + property] = property;
            this.queryStatement.ExpressionAttributeValues[':' + property] = value;
            return this;
        }
        if (value === undefined) {
            value = operator;
            operator = '=';
        }
        if (operator === undefined) {
            operator = '=';
        }
        if (value === undefined) {
            value = '';
        }

        if (operator === 'HAS_ANCESTOR') {
            throw new Error('HAS_ANCESTOR operator is not supported in query filters.');
        }
        if (!!(this.queryStatement.FilterExpression))
            this.queryStatement.FilterExpression += ' AND ';

        this.queryStatement.FilterExpression += '#' + property + operator + ':' + property;
        this.queryStatement.ExpressionAttributeNames['#' + property] = property;
        this.queryStatement.ExpressionAttributeValues[':' + property] = value;
        return this;
    }

    start(start: string | Buffer): IJournalQueryModel {
        if (start instanceof Buffer) {
            throw new Error('Type \'Buffer\' is not supported for DynamoDB Continuation while paging.');
        }
        // this.queryStatement.ExclusiveStartKey = start as DynamoDB.String;
        console.log('NOT SUPPOR aws start createQuery ' + start);
        return this;
    }

    limit(n: number): IJournalQueryModel {
        this.queryStatement.Limit = n;
        return this;
    }

    groupBy(fieldNames: string | string[]): IJournalQueryModel {
        console.log('NOT SUPPORT aws groupBy createQuery ' + fieldNames);
        return this;
    }

    select(fieldNames: string | string[]): IJournalQueryModel {
        if (this.queryStatement.ProjectionExpression.length >= 1)
            this.queryStatement.ProjectionExpression += ',';
        if (typeof fieldNames === 'string') {
            this.queryStatement.ProjectionExpression += fieldNames;
        } else {
            this.queryStatement.ProjectionExpression += fieldNames.join(',');
        }
        return this;
    }
    public getQueryStatement(tableName: string): ScanInput {
        //delete empty objects in query parameters
        if (Object.entries(this.queryStatement.ExpressionAttributeNames).length === 0) {
            delete this.queryStatement.ExpressionAttributeNames;
            delete this.queryStatement.ExpressionAttributeValues;
        }
        if (this.queryStatement.FilterExpression.length === 0)
            delete this.queryStatement.FilterExpression;

        this.queryStatement.TableName = AWSConfig.SERVICE_ENV + '-SeismicStore.' + tableName;
        return this.queryStatement;
    }
}