Commit 029cfba8 authored by Yunhua Koglin's avatar Yunhua Koglin Committed by Rucha Deshpande
Browse files

fix the storage interfaces.

commit 127c9881 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Jan 22 2021 10:11:54 GMT-0600 (Central Standard Time) 

    fix the storage


commit ab011a83 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Thu Jan 21 2021 15:29:00 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit b08276eb 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Thu Jan 21 2021 14:08:59 GMT-0600 (Central Standard Time) 

    fix the table name


commit 3e498ecf 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Thu Jan 21 2021 14:00:37 GMT-0600 (Central Standard Time) 

    add ENVIROMENT env


commit a0794cf7 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Wed Jan 20 2021 16:59:17 GMT-0600 (Central Standard Time) 

    add entry for app table, fix id for dataset


commit 7013750b 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Wed Jan 20 2021 14:56:20 GMT-0600 (Central Standard Time) 

    fix Journal, JournalTransaction and JournalQuery in dynamodb


commit a916a370 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Wed Jan 20 2021 14:54:11 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit bd12e513 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Sun Jan 17 2021 23:18:41 GMT-0600 (Central Standard Time) 

    Revert "update packages with aws-sdk"

This reverts commit fa7ba482aba74830daee7935a46daaf2cbdc3d33.


commit 64740dd9 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Sun Jan 17 2021 22:37:08 GMT-0600 (Central Standard Time) 

    add storage and dynamodb


commit 739900b4 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Jan 15 2021 16:16:21 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit 5f969e3a 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Jan 15 2021 15:47:49 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit 0ac28b3b 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Wed Jan 13 2021 11:35:28 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit fa7ba482 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Tue Jan 12 2021 09:26:08 GMT-0600 (Central Standard Time) 

    update packages with aws-sdk
parent 6d19ed38
......@@ -19,6 +19,9 @@ import { AWSConfig } from './config';
import {AWSSSMhelper} from './ssmhelper';
import {AWSSTShelper} from './stshelper';
import DynamoDB from 'aws-sdk/clients/dynamodb';
import aws from 'aws-sdk';
@CredentialsFactory.register('aws')
export class AWSCredentials extends AbstractCredentials {
......@@ -42,6 +45,24 @@ export class AWSCredentials extends AbstractCredentials {
return undefined;
}
async getBucketFolder(folder:string): Promise<string> {
const tableName = AWSConfig.AWS_ENVIRONMENT+'-SeismicStore.'+AWSConfig.SUBPROJECTS_KIND;
const params = {
TableName: tableName,
Key: {
'id': {S: folder}
}
};
const db = new DynamoDB({});
const data = await db.getItem(params).promise();
const ret = aws.DynamoDB.Converter.unmarshall(data.Item);
if (Object.keys(ret).length === 0)
return undefined;
else
return ret['gcs_bucket'];
}
async getUserCredentials(subject: string): Promise<IAccessTokenModel> {
//subject = tenantName:subprojectName:1 ==> readOnly true
//subject = tenantName:subprojectName:0 ==> readOnly false
......@@ -56,7 +77,9 @@ export class AWSCredentials extends AbstractCredentials {
var flagUpload=true;
const keypath = tenant+'/'+subproject;
const keypath = await this.getBucketFolder(tenant+':'+subproject);
// tslint:disable-next-line:triple-equals
if(readOnly ==='1') { // readOnly True
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/download-role-arn')
......
......@@ -17,76 +17,95 @@ import { AbstractStorage, StorageFactory } from '../../storage';
import { TenantModel } from '../../../services/tenant';
import AWS from 'aws-sdk/global';
import S3 from "aws-sdk/clients/s3";
import {AWSSSMhelper} from './ssmhelper';
@StorageFactory.register('aws')
export class AWSStorage extends AbstractStorage {
private projectID: string;
private BUCKET_PREFIX = 'ss-' + AWSConfig.SERVICE_ENV;
private s3: S3;
private static bucketName: string;
private s3: S3; // S3 service object
public constructor(tenant: TenantModel) {
super();
this.projectID = tenant.gcpid;
AWS.config.update({ region: AWSConfig.AWS_REGION });
// Create S3 service object
this.s3 = new S3({ apiVersion: '2006-03-01' });
}
// generate a random bucket name
// get the bucketName from SSM
private async getBucketName(): Promise<void> {
if (AWSStorage.bucketName !== undefined)
return;
const awsSSMHelper = new AWSSSMhelper();
AWSStorage.bucketName = await awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name');
console.log(AWSStorage.bucketName);
}
private fixFolderFormat(folder: string): string {
if (folder) {
folder += '/';
while (folder.indexOf('//') !== -1) {
folder = folder.replace('//', '/');
}
}
return folder;
}
// generate a random bucket name, for aws, a random folder name
public randomBucketName(): string {
let suffix = Math.random().toString(36).substring(2, 16);
suffix = suffix + Math.random().toString(36).substring(2, 16);
suffix = suffix.substr(0, 16);
return this.BUCKET_PREFIX + '-' + suffix;
return suffix;
}
// Create a new bucket
// Create a new bucket, for aws, create a folder with folderName
public async createBucket(
bucketName: string,
folderName: string,
location: string, storageClass: string,
adminACL: string, editorACL: string, viewerACL: string): Promise<void> {
const create_bucket_params = {
Bucket: bucketName,
await this.getBucketName();
const params = {
Bucket: AWSStorage.bucketName,
Key: folderName + '/',
Body: ''
};
try {
await this.s3.createBucket(create_bucket_params).promise();
await this.s3.putObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
}
// var params = {
// Bucket: bucketName,
// GrantFullControl: adminACL,
// GrantWrite: editorACL,
// GrantRead: viewerACL
// };
// await this.s3.putBucketAcl(params).promise();
}
// Delete a bucket
public async deleteBucket(bucketName: string, force = false): Promise<void> {
// Delete a bucket, for aws, delete folder folderName
public async deleteBucket(folderName: string, force = false): Promise<void> {
await this.getBucketName();
if (force) {
await this.deleteFiles(bucketName);
await this.deleteFiles(folderName);
}
const delete_bucket_params = { Bucket: bucketName };
const params = {
Bucket: AWSStorage.bucketName,
Key: folderName + '/'
};
try {
await this.s3.deleteBucket(delete_bucket_params).promise();
await this.s3.deleteObject(params).promise();
} catch (err) {
console.log(err.code + ": " + err.message);
}
}
// Delete all files in a bucket
public async deleteFiles(bucketName: string): Promise<void> {
console.log("start to delete all files in " + bucketName);
const params = { Bucket: bucketName };
// Delete all files in a bucket, for aws, delete all files in the folder
public async deleteFiles(folderName: string): Promise<void> {
await this.getBucketName();
const params = {
Bucket: AWSStorage.bucketName,
Prefix: folderName + '/'
};
const listedObjects = await this.s3.listObjectsV2(params).promise();
if (listedObjects.Contents.length === 0)
return;
const deleteParams = {
Bucket: bucketName,
Bucket: AWSStorage.bucketName,
Delete: { Objects: [] }
};
......@@ -97,12 +116,17 @@ export class AWSStorage extends AbstractStorage {
await this.s3.deleteObjects(deleteParams).promise();
if (listedObjects.IsTruncated) //continue delete files as there are more...
await this.deleteFiles(bucketName);
await this.deleteFiles(folderName);
}
// save an object/file to a bucket, object name contains path
public async saveObject(bucketName: string, objectName: string, data: string): Promise<void> {
const params = { Bucket: bucketName, Key: objectName, Body: data };
// save an object/file to a bucket
public async saveObject(folderName: string, objectName: string, data: string): Promise<void> {
await this.getBucketName();
const params = {
Bucket: AWSStorage.bucketName,
Key: folderName + '/' + objectName,
Body: data
};
try {
this.s3.putObject(params).promise();
} catch (err) {
......@@ -110,9 +134,13 @@ export class AWSStorage extends AbstractStorage {
}
}
// delete an object from a bucket
public async deleteObject(bucketName: string, objectName: string): Promise<void> {
const params = { Bucket: bucketName, Key: objectName };
// delete an object from a bucket
public async deleteObject(folderName: string, objectName: string): Promise<void> {
await this.getBucketName();
const params = {
Bucket: AWSStorage.bucketName,
Key: folderName + '/' + objectName
};
try {
this.s3.deleteObject(params).promise();
} catch (err) {
......@@ -120,17 +148,20 @@ export class AWSStorage extends AbstractStorage {
}
}
// delete multiple objects, prefix should end with /
public async deleteObjects(bucketName: string, prefix: string, async: boolean = false): Promise<void> {
console.log("start deleteObjects in " + prefix + " in bucket " + bucketName);
// delete multiple objects
public async deleteObjects(folderName: string, prefix: string, async: boolean = false): Promise<void> {
await this.getBucketName();
const params = { Bucket: bucketName, Prefix: prefix };
const params = {
Bucket: AWSStorage.bucketName,
Prefix: folderName + '/' + prefix
};
const listedObjects = await this.s3.listObjectsV2(params).promise();
if (listedObjects.Contents.length === 0) return;
const deleteParams = {
Bucket: bucketName,
Bucket: AWSStorage.bucketName,
Delete: { Objects: [] }
};
......@@ -141,49 +172,48 @@ export class AWSStorage extends AbstractStorage {
await this.s3.deleteObjects(deleteParams).promise();
if (listedObjects.IsTruncated) //continue delete files as there are more...
await this.deleteObjects(bucketName, prefix);
await this.deleteObjects(folderName, prefix, async);
}
// copy multiple objects (skip the dummy file)
public async copy(bucketIn: string, prefixIn: string, bucketOut: string, prefixOut: string, ownerEmail: string) {
if (prefixIn) {
prefixIn += '/';
while (prefixIn.indexOf('//') !== -1) {
prefixIn = prefixIn.replace('//', '/');
}
}
if (prefixOut) {
prefixOut += '/';
while (prefixOut.indexOf('//') !== -1) {
prefixOut = prefixOut.replace('//', '/');
}
}
public async copy(folderIn: string, prefixIn: string, folderOut: string, prefixOut: string, ownerEmail: string) {
await this.getBucketName();
prefixIn = this.fixFolderFormat(prefixIn);
prefixOut = this.fixFolderFormat(prefixOut);
const copyCalls = [];
const params = { Bucket: bucketIn, Prefix: prefixIn };
const params = {
Bucket: AWSStorage.bucketName,
Prefix: folderIn + '/' + prefixIn
};
const files = await this.s3.listObjects(params).promise();
for (const file of files[0]) {
for (const file of files['Contents']) {
var newKey = file.Key.replace(folderIn, folderOut);
newKey = newKey.replace(prefixIn, prefixOut);
const params = {
Bucket: bucketOut,
CopySource: bucketIn + '/' + file.Key,
Key: file.Key.replace(prefixIn, prefixOut)
Bucket: AWSStorage.bucketName,
CopySource: file.Key,
Key: newKey
};
copyCalls.push(this.s3.copyObject(params));
}
await Promise.all(copyCalls);
}
// check if a bucket exist
public async bucketExists(bucketName: string): Promise<boolean> {
const bucket_params = { Bucket: bucketName };
try {
await this.s3.headBucket(bucket_params).promise();
return true;
} catch (err) {
if (err.statusCode === 404) { //404 if the bucket does not exist
return false;
}
}
// check if a bucket exist, for aws, check if folder in the bucket
// folderName is a string without / at the end
public async bucketExists(folderName: string): Promise<boolean> {
await this.getBucketName();
const params = {
Bucket: AWSStorage.bucketName,
Prefix: folderName
};
const listedObjects = await this.s3.listObjectsV2(params).promise();
if (listedObjects.Contents.length === 0)
return false;
return true;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment