Commit 48417707 authored by Rucha Deshpande's avatar Rucha Deshpande
Browse files

Merge branch 'feat/aws-tslint-fixes' into 'master'

run tslint on source code

See merge request !46
parents c90e8280 ff58541f
Pipeline #33395 passed with stages
in 6 minutes and 19 seconds
......@@ -50,6 +50,7 @@ phases:
- mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- printenv
- node_modules/.bin/tslint -c tslint.json 'src/cloud/providers/aws/**/*.ts'
- echo "Building seismic-store-service"
- npm run build
......
......@@ -22,7 +22,7 @@ export class AWSConfig extends Config {
public static AWS_REGION: string;
public static AWS_ENVIRONMENT: string;
public static AWS_BUCKET: string;
//Logger
// Logger
public static LOGGER_LEVEL;
// max len for a group name in DE
public static DES_GROUP_CHAR_LIMIT = 256;
......@@ -37,8 +37,8 @@ export class AWSConfig extends Config {
const awsSSMHelper = new AWSSSMhelper();
AWSConfig.AWS_BUCKET = await awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name');
//Logger
// Logger
AWSConfig.LOGGER_LEVEL = process.env.LOGGER_LEVEL || 'info';
Config.initServiceConfiguration({
......
......@@ -57,27 +57,28 @@ export class AWSCredentials extends AbstractCredentials {
const data = await db.getItem(params).promise();
const ret = aws.DynamoDB.Converter.unmarshall(data.Item);
if (Object.keys(ret).length === 0){
console.log("error to get folder: "+folder+"\n");
// tslint:disable-next-line:no-console
console.log('error to get folder: '+folder+'\n');
return undefined;
}
else{
const vars = ret['gcs_bucket'].split("$$");
const vars = ret['gcs_bucket'].split('$$');
return vars[1];
}
}
public async getStorageCredentials(
tenant: string, subproject: string,
bucket: string, readonly: boolean, _partition: string): Promise<IAccessTokenModel> {
const s3bucket = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name')
const expDuration = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/temp-cred-expiration-duration')
var roleArn='';
var credentials='';
var flagUpload=true;
let roleArn='';
let credentials='';
let flagUpload=true;
const keypath = await this.getBucketFolder(tenant+':'+subproject);
// tslint:disable-next-line:triple-equals
if(readonly ) { // readOnly True
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/download-role-arn')
......@@ -87,7 +88,7 @@ export class AWSCredentials extends AbstractCredentials {
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/upload-role-arn')
flagUpload = true;
}
credentials = await this.awsSTSHelper.getCredentials(s3bucket,keypath,roleArn,flagUpload,expDuration);
const result = {
......
......@@ -27,7 +27,7 @@ const converter = aws.DynamoDB.Converter;
export class AWSDynamoDbDAO extends AbstractJournal {
public KEY = Symbol('id');
private dataPartition: string; //tenant name
private dataPartition: string; // tenant name
private tenant: TenantModel;
public constructor(tenant: TenantModel) {
super();
......@@ -42,27 +42,29 @@ export class AWSDynamoDbDAO extends AbstractJournal {
}
for (const entity of datasetEntity) {
const item = entity.data;
//id attribute is used by aws as partitionKey.
//For tenant, id = name, for subproject, id=tenant:name; for dataset, id=tenant:subproject:name:path; for app, id=tenant:email
var strs = entity.key.partitionKey.split(':');
if(entity.key.table_kind == AWSConfig.DATASETS_KIND && strs.length == 2){
//fill in data name and path to the key
// id attribute is used by aws as partitionKey.
// For tenant, id = name, for subproject, id=tenant:name;
// for dataset, id=tenant:subproject:name:path; for app, id=tenant:email
const strs = entity.key.partitionKey.split(':');
if(entity.key.tableKind === AWSConfig.DATASETS_KIND && strs.length === 2){
// fill in data name and path to the key
entity.key.partitionKey = entity.key.partitionKey+':'+item.name+':'+item.path;
}
if(entity.key.table_kind == AWSConfig.APPS_KIND){
item['tenant'] = strs[0]; //add tenant entry for App table
if(entity.key.tableKind === AWSConfig.APPS_KIND){
item['tenant'] = strs[0]; // add tenant entry for App table
}
item['id'] = entity.key.partitionKey;
if (entity.ctag) {
item['ctag'] = entity.ctag;
}
//save extra info as this property will be consumed by the service to identify a data record
// save extra info as this property will be consumed by the service to identify a data record
item[this.KEY.toString()] = entity.key;
const itemMarshall = converter.marshall(item);
console.log('from table ' + entity.key.table_name + ' save ' + JSON.stringify(itemMarshall));
// tslint:disable-next-line:no-console
console.log('from table ' + entity.key.tableName + ' save ' + JSON.stringify(itemMarshall));
const para = {
TableName: entity.key.table_name,
TableName: entity.key.tableName,
Item: itemMarshall
};
const db = new DynamoDB({});
......@@ -74,9 +76,10 @@ export class AWSDynamoDbDAO extends AbstractJournal {
const item = { 'id': key.partitionKey };
const itemMarshall = converter.marshall(item);
console.log('from table ' + key.table_name + ' get ' + JSON.stringify(itemMarshall));
// tslint:disable-next-line:no-console
console.log('from table ' + key.tableName + ' get ' + JSON.stringify(itemMarshall));
const params = {
TableName: key.table_name,
TableName: key.tableName,
Key: itemMarshall
};
const db = new DynamoDB({});
......@@ -85,7 +88,7 @@ export class AWSDynamoDbDAO extends AbstractJournal {
if (Object.keys(ret).length === 0)
return [undefined];
else {
//remove aws specific attribute id
// remove aws specific attribute id
delete ret['id'];
// to pass integration test
delete ret[this.KEY.toString()];
......@@ -96,9 +99,10 @@ export class AWSDynamoDbDAO extends AbstractJournal {
public async delete(key: any): Promise<void> {
const item = { 'id': key.partitionKey };
const itemMarshall = converter.marshall(item);
console.log('from table ' + key.table_name + ' delete ' + JSON.stringify(itemMarshall));
// tslint:disable-next-line:no-console
console.log('from table ' + key.tableName + ' delete ' + JSON.stringify(itemMarshall));
const params = {
TableName: key.table_name,
TableName: key.tableName,
Key: itemMarshall
};
const db = new DynamoDB({});
......@@ -113,16 +117,17 @@ export class AWSDynamoDbDAO extends AbstractJournal {
const dbQuery = (query as AWSDynamoDbQuery);
const statement = dbQuery.getQueryStatement(dbQuery.kind);
// tslint:disable-next-line:no-console
console.log('query ' + JSON.stringify(statement));
const db = new DynamoDB.DocumentClient();
var scanResults = [];
var items: PromiseResult<DynamoDB.DocumentClient.ScanOutput, AWS.AWSError>;
let scanResults = [];
let items: PromiseResult<DynamoDB.DocumentClient.ScanOutput, AWS.AWSError>;
do {
items = await db.scan(statement).promise();
const results = items.Items.map(result => {
var ret = {};
let ret = {};
ret = result;
//update object property for service (dao.ts) to consume
// update object property for service (dao.ts) to consume
if (ret[this.KEY.toString()]) {
ret[this.KEY] = result[this.KEY.toString()];
}
......@@ -130,28 +135,28 @@ export class AWSDynamoDbDAO extends AbstractJournal {
});
scanResults = scanResults.concat(results);
statement.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey !== "undefined");
} while (typeof items.LastEvaluatedKey !== 'undefined');
return Promise.resolve([scanResults, { endCursor: items.LastEvaluatedKey }]);
}
public createKey(specs: any): object {
const table_kind = specs.path[0];
const name = specs.path[1]; //our key
var partitionKey = name; // partitionKey
const tableKind = specs.path[0];
const name = specs.path[1]; // our key
let partitionKey = name; // partitionKey
var strs = specs.namespace.split('-');
if (table_kind === AWSConfig.SUBPROJECTS_KIND) {
partitionKey = strs[strs.length - 1] + ':' + partitionKey; //tenant:subprojet for id
const strs = specs.namespace.split('-');
if (tableKind === AWSConfig.SUBPROJECTS_KIND) {
partitionKey = strs[strs.length - 1] + ':' + partitionKey; // tenant:subprojet for id
}
if (table_kind === AWSConfig.DATASETS_KIND) {
partitionKey = strs[strs.length - 2] + ':' + strs[strs.length - 1]; //tenant:subprojet for id
if (tableKind === AWSConfig.DATASETS_KIND) {
partitionKey = strs[strs.length - 2] + ':' + strs[strs.length - 1]; // tenant:subprojet for id
}
if (table_kind === AWSConfig.APPS_KIND) {
partitionKey = strs[strs.length - 1] + ':' + name; //tenant:subprojet for id
if (tableKind === AWSConfig.APPS_KIND) {
partitionKey = strs[strs.length - 1] + ':' + name; // tenant:subprojet for id
}
const table_name = AWSConfig.AWS_ENVIRONMENT + '-' + 'SeismicStore.' + specs.path[0];
return { table_name, name, table_kind, partitionKey };
const tableName = AWSConfig.AWS_ENVIRONMENT + '-' + 'SeismicStore.' + specs.path[0];
return { tableName, name, tableKind, partitionKey };
}
public getTransaction(): IJournalTransaction {
......@@ -187,33 +192,39 @@ export class AWSDynamoDbTransactionDAO extends AbstractJournalTransaction {
}
public async save(entity: any): Promise<void> {
// tslint:disable-next-line:no-console
console.log('aws Transaction Save ' + JSON.stringify(entity));
this.queuedOperations.push(new AWSDynamoDbTransactionOperation('save', entity));
await Promise.resolve();
}
public async get(key: any): Promise<[any | any[]]> {
// tslint:disable-next-line:no-console
console.log('aws Transaction get ' + JSON.stringify(key));
return await this.owner.get(key);
}
public async delete(key: any): Promise<void> {
// tslint:disable-next-line:no-console
console.log('aws Transaction delete ' + JSON.stringify(key));
this.queuedOperations.push(new AWSDynamoDbTransactionOperation('delete', key));
await Promise.resolve();
}
public createQuery(namespace: string, kind: string): IJournalQueryModel {
// tslint:disable-next-line:no-console
console.log('aws Transaction createQuery ' + namespace + kind);
return this.owner.createQuery(namespace, kind);
}
public async runQuery(query: IJournalQueryModel): Promise<[any[], { endCursor?: string }]> {
// tslint:disable-next-line:no-console
console.log('aws Transaction runQuery ' + JSON.stringify(query));
return await this.owner.runQuery(query);
}
public async run(): Promise<void> {
// tslint:disable-next-line:no-console
console.log('aws Transaction run ');
if (this.queuedOperations.length) {
await Promise.reject('Transaction is already in use.');
......@@ -225,12 +236,14 @@ export class AWSDynamoDbTransactionDAO extends AbstractJournalTransaction {
}
public async rollback(): Promise<void> {
// tslint:disable-next-line:no-console
console.log('aws Transaction rollback ');
this.queuedOperations = [];
return Promise.resolve();
}
public async commit(): Promise<void> {
// tslint:disable-next-line:no-console
console.log('aws Transaction commit ');
for (const operation of this.queuedOperations) {
if (operation.type === 'save') {
......@@ -259,7 +272,8 @@ export class AWSDynamoDbQuery implements IJournalQueryModel {
public constructor(namespace: string, kind: string) {
this.namespace = namespace;
this.kind = kind;
this.queryStatement = { TableName: kind, FilterExpression: '', ExpressionAttributeNames: {}, ExpressionAttributeValues: {}, ProjectionExpression:'' };
this.queryStatement = { TableName: kind, FilterExpression: '',
ExpressionAttributeNames: {}, ExpressionAttributeValues: {}, ProjectionExpression:'' };
}
public namespace: string;
public kind: string;
......@@ -274,14 +288,14 @@ export class AWSDynamoDbQuery implements IJournalQueryModel {
if (!!(this.queryStatement.FilterExpression)) {
this.queryStatement.FilterExpression += ' AND ';
}
var i = 0;
var propertyValue = property;
let i = 0;
let propertyValue = property;
while (true){
if (this.queryStatement.ExpressionAttributeValues[':'+propertyValue] != undefined){ //already used
propertyValue = propertyValue+i; //gtag0, gtag1, gtag2....
if (this.queryStatement.ExpressionAttributeValues[':'+propertyValue] !== undefined){ // already used
propertyValue = propertyValue+i; // gtag0, gtag1, gtag2....
i++;
}else {
break; //break true
break; // break true
}
}
this.queryStatement.FilterExpression += 'contains(#' + property + ',:' + propertyValue + ')';
......@@ -328,6 +342,7 @@ export class AWSDynamoDbQuery implements IJournalQueryModel {
if (start instanceof Buffer) {
throw new Error('Type \'Buffer\' is not supported for DynamoDB Continuation while paging.');
}
// tslint:disable-next-line:no-console
console.log('NOT SUPPOR aws start createQuery ' + start);
return this;
}
......@@ -338,6 +353,7 @@ export class AWSDynamoDbQuery implements IJournalQueryModel {
}
groupBy(fieldNames: string | string[]): IJournalQueryModel {
// tslint:disable-next-line:no-console
console.log('NOT SUPPORT aws groupBy createQuery ' + fieldNames);
return this;
}
......@@ -357,45 +373,48 @@ export class AWSDynamoDbQuery implements IJournalQueryModel {
}
public getQueryStatement(tableName: string): ScanInput {
//since we have one table for all datasets, we need to add more filters to return dataset specific for that tenant/subproject
var strs = this.namespace.split('-');
if (this.kind === AWSConfig.DATASETS_KIND || this.kind === AWSConfig.SUBPROJECTS_KIND) { //one table, filter on tenant
// since we have one table for all datasets, we need to
// add more filters to return dataset specific for that tenant/subproject
const strs = this.namespace.split('-');
if (this.kind === AWSConfig.DATASETS_KIND || this.kind === AWSConfig.SUBPROJECTS_KIND) {
// one table, filter on tenant
if (!!(this.queryStatement.FilterExpression)) {
this.queryStatement.FilterExpression += ' AND ';
}
const t_property: string = 'tenant';
var value = {}; value = this.kind === AWSConfig.DATASETS_KIND ? strs[strs.length - 2] : strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
const tProperty: string = 'tenant';
let value = {};
value = this.kind === AWSConfig.DATASETS_KIND ? strs[strs.length - 2] : strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + tProperty + '=' + ':' + tProperty;
this.queryStatement.ExpressionAttributeNames['#' + tProperty] = tProperty;
this.queryStatement.ExpressionAttributeValues[':' + tProperty] = value;
}
if (this.kind === AWSConfig.DATASETS_KIND) { // one table, filter on subproject too
this.queryStatement.FilterExpression += ' AND ';
const t_property: string = 'subproject';
var value = {}; value = strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
const tProperty: string = 'subproject';
let value = {}; value = strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + tProperty + '=' + ':' + tProperty;
this.queryStatement.ExpressionAttributeNames['#' + tProperty] = tProperty;
this.queryStatement.ExpressionAttributeValues[':' + tProperty] = value;
}
if (this.kind === AWSConfig.APPS_KIND) { // one table, filter on tenant
if (!!(this.queryStatement.FilterExpression)) {
this.queryStatement.FilterExpression += ' AND ';
}
const t_property: string = 'tenant';
var value = {}; value = strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + t_property + '=' + ':' + t_property;
this.queryStatement.ExpressionAttributeNames['#' + t_property] = t_property;
this.queryStatement.ExpressionAttributeValues[':' + t_property] = value;
const tProperty: string = 'tenant';
let value = {}; value = strs[strs.length - 1];
this.queryStatement.FilterExpression += '#' + tProperty + '=' + ':' + tProperty;
this.queryStatement.ExpressionAttributeNames['#' + tProperty] = tProperty;
this.queryStatement.ExpressionAttributeValues[':' + tProperty] = value;
}
if (this.queryStatement.FilterExpression.length === 0)
delete this.queryStatement.FilterExpression;
if (this.queryStatement.ProjectionExpression.length === 0)
delete this.queryStatement.ProjectionExpression;
//delete empty objects in query parameters
// delete empty objects in query parameters
if (Object.entries(this.queryStatement.ExpressionAttributeNames).length === 0) {
delete this.queryStatement.ExpressionAttributeNames;
delete this.queryStatement.ExpressionAttributeValues;
......
......@@ -21,19 +21,19 @@ import { AWSConfig } from './config';
export class AwsLogger extends AbstractLogger {
public info(data: any): void {
logger.info(data);
logger.info(data);
}
public debug(data: any): void {
logger.debug(data);
logger.debug(data);
}
public error(data: any): void {
logger.error(data);
}
public metric(key:string,data: any): void {
logger.info("No Metric");
logger.info('No Metric');
}
}
......
......@@ -38,6 +38,7 @@ export class AWSSSMhelper {
// console.log(data.Parameter.Value);
return data.Parameter.Value;
} catch (err) {
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
}
}
......
......@@ -16,7 +16,7 @@ import { AWSConfig } from './config';
import { AbstractStorage, StorageFactory } from '../../storage';
import { TenantModel } from '../../../services/tenant';
import AWS from 'aws-sdk/global';
import S3 from "aws-sdk/clients/s3";
import S3 from 'aws-sdk/clients/s3';
@StorageFactory.register('aws')
export class AWSStorage extends AbstractStorage {
......@@ -38,22 +38,22 @@ export class AWSStorage extends AbstractStorage {
return folder;
}
// generate a random bucket name, for aws, a random folder name
// generate a random bucket name, for aws, a random folder name
public randomBucketName(): string {
let suffix = Math.random().toString(36).substring(2, 16);
suffix = suffix + Math.random().toString(36).substring(2, 16);
suffix = suffix.substr(0, 16);
return AWSConfig.AWS_BUCKET+"$$"+suffix;
return AWSConfig.AWS_BUCKET+'$$'+suffix;
}
//whenever ask for a bucket, we return bucketName$$folderName for that subprject
//this function return the real folderName by remove bucketName$$ at the front of folderName
// whenever ask for a bucket, we return bucketName$$folderName for that subprject
// this function return the real folderName by remove bucketName$$ at the front of folderName
public getFolder(folderName:string): string {
var start = AWSConfig.AWS_BUCKET.length+2;
var str = folderName.substr(start);
const start = AWSConfig.AWS_BUCKET.length+2;
const str = folderName.substr(start);
return str;
}
// Create a new bucket, for aws, create a folder with folderName
public async createBucket(
folderName: string,
......@@ -67,11 +67,12 @@ export class AWSStorage extends AbstractStorage {
try {
await this.s3.putObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
}
}
// Delete a bucket, for aws, delete folder folderName
// Delete a bucket, for aws, delete folder folderName
public async deleteBucket(folderName: string, force = false): Promise<void> {
if (force) {
await this.deleteFiles(folderName);
......@@ -84,7 +85,8 @@ export class AWSStorage extends AbstractStorage {
try {
await this.s3.deleteObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
}
}
......@@ -110,7 +112,7 @@ export class AWSStorage extends AbstractStorage {
await this.s3.deleteObjects(deleteParams).promise();
if (listedObjects.IsTruncated) //continue delete files as there are more...
if (listedObjects.IsTruncated) // continue delete files as there are more...
await this.deleteFiles(folderName);
}
......@@ -125,11 +127,12 @@ export class AWSStorage extends AbstractStorage {
try {
this.s3.putObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
}
}
// delete an object from a bucket
// delete an object from a bucket
public async deleteObject(folderName: string, objectName: string): Promise<void> {
const folder = this.getFolder(folderName);
const params = {
......@@ -139,7 +142,8 @@ export class AWSStorage extends AbstractStorage {
try {
this.s3.deleteObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
}
}
......@@ -165,7 +169,7 @@ export class AWSStorage extends AbstractStorage {
await this.s3.deleteObjects(deleteParams).promise();
if (listedObjects.IsTruncated) //continue delete files as there are more...
if (listedObjects.IsTruncated) // continue delete files as there are more...
await this.deleteObjects(folderName, prefix, async);
}
......@@ -186,14 +190,14 @@ export class AWSStorage extends AbstractStorage {
const files = await this.s3.listObjects(params).promise();
for (const file of files['Contents']) {
var newKey = file.Key.replace(realFolderIn, realFolderOut);
let newKey = file.Key.replace(realFolderIn, realFolderOut);
newKey = newKey.replace(folderIn, prefixOut);
const params = {
const param = {
Bucket: AWSConfig.AWS_BUCKET,
CopySource: file.Key,
Key: newKey
};
copyCalls.push(this.s3.copyObject(params));
copyCalls.push(this.s3.copyObject(param));
}
await Promise.all(copyCalls);
}
......
......@@ -27,43 +27,45 @@ export class AWSSTShelper {
this.sts = new STS({apiVersion: '2014-11-06'});
}
public async getCredentials(bucketName: string, keypath: string, roleArn: string, flagUpload: boolean, exp: string): Promise<string> {
var policy;
public async getCredentials(bucketName: string, keypath: string,
roleArn: string, flagUpload: boolean, exp: string): Promise<string> {
let policy;
if(flagUpload === true)
policy = this.createUploadPolicy(bucketName,keypath);
else
policy = this.createDownloadPolicy(bucketName,keypath);
var expDuration: number = +exp;
const expDuration: number = +exp;
let stsParams = {
ExternalId: "OSDUAWS",
const stsParams = {
ExternalId: 'OSDUAWS',
Policy: policy,
RoleArn: roleArn,
RoleSessionName: "OSDUAWSAssumeRoleSession",
RoleSessionName: 'OSDUAWSAssumeRoleSession',
DurationSeconds: expDuration
};
const roleCreds = await this.sts.assumeRole(stsParams).promise();
const tempCreds= roleCreds.Credentials.AccessKeyId+':'+roleCreds.Credentials.SecretAccessKey+':'+roleCreds.Credentials.SessionToken;