Commit de591c12 authored by Yunhua Koglin's avatar Yunhua Koglin Committed by Rucha Deshpande
Browse files

fix e2e tests and update server with the changes from upstream

commit a0f514aa 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Tue Mar 02 2021 10:21:51 GMT-0600 (Central Standard Time) 

    change logger level


commit b992bb32 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Mon Mar 01 2021 21:52:22 GMT-0600 (Central Standard Time) 

    add server changes


commit 1e9262d6 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Mon Mar 01 2021 10:14:32 GMT-0600 (Central Standard Time) 

    merge dev


commit 9f3726cd 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Feb 26 2021 11:26:42 GMT-0600 (Central Standard Time) 

    merge


commit ed984111 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Tue Feb 23 2021 14:42:04 GMT-0600 (Central Standard Time) 

    add logger


commit 304d0bc5 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Feb 11 2021 11:42:34 GMT-0600 (Central Standard Time) 

    Fix DES_SERVICE_HOST changes

commit c716e4a7 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Feb 11 2021 11:38:42 GMT-0600 (Central Standard Time) 

    Fix DES_SERVICE_HOST changes



commit 7444eac9 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Tue Feb 23 2021 12:30:49 GMT-0600 (Central Standard Time) 

    add the changes


commit 0c3b13cd 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Feb 05 2021 19:40:48 GMT-0600 (Central Standard Time) 

    fix get folder by removing prefix

commit ffbd1c60 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Feb 05 2021 18:19:55 GMT-0600 (Central Standard Time) 

    fix getting folder


commit 76c955cd 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Fri Feb 05 2021 15:04:31 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny


commit 32ff3ed8 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Thu Feb 04 2021 17:35:53 GMT-0600 (Central Standard Time) 

    Merge branch 'dev' of codecommit://os-seismic-store-service into kogliny
parent 7146d5c6
......@@ -22,7 +22,8 @@ export class AWSConfig extends Config {
public static AWS_REGION: string;
public static AWS_ENVIRONMENT: string;
public static AWS_BUCKET: string;
//Logger
public static LOGGER_LEVEL;
// max len for a group name in DE
public static DES_GROUP_CHAR_LIMIT = 256;
......@@ -37,6 +38,9 @@ export class AWSConfig extends Config {
const awsSSMHelper = new AWSSSMhelper();
AWSConfig.AWS_BUCKET = await awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name');
//Logger
AWSConfig.LOGGER_LEVEL = process.env.LOGGER_LEVEL || 'info';
Config.initServiceConfiguration({
SERVICE_ENV: process.env.SERVICE_ENV,
SERVICE_PORT: +process.env.PORT || 5000,
......
......@@ -65,44 +65,37 @@ export class AWSCredentials extends AbstractCredentials {
return vars[1];
}
}
async getUserCredentials(subject: string): Promise<IAccessTokenModel> {
//subject = tenantName:subprojectName:1 ==> readOnly true
//subject = tenantName:subprojectName:0 ==> readOnly false
const s3bucket = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name')
const expDuration = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/temp-cred-expiration-duration')
const vars = subject.split(':')
const tenant = vars[0];
const subproject = vars[1];
const readOnly = vars[2];
var roleArn='';
var credentials='';
var flagUpload=true;
const keypath = await this.getBucketFolder(tenant+':'+subproject);
// tslint:disable-next-line:triple-equals
if(readOnly ==='1') { // readOnly True
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/download-role-arn')
flagUpload = false;
} else if (readOnly ==='0') // readOnly False
{
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/upload-role-arn')
flagUpload = true;
}
credentials = await this.awsSTSHelper.getCredentials(s3bucket,keypath,roleArn,flagUpload,expDuration);
const result = {
access_token: credentials,
expires_in: 3599,
token_type: 'Bearer',
};
return result;
public async getStorageCredentials(
tenant: string, subproject: string,
bucket: string, readonly: boolean, _partition: string): Promise<IAccessTokenModel> {
const s3bucket = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/seismic-s3-bucket-name')
const expDuration = await this.awsSSMHelper.getSSMParameter('/osdu/'+AWSConfig.AWS_ENVIRONMENT+'/seismic-store/temp-cred-expiration-duration')
var roleArn='';
var credentials='';
var flagUpload=true;
const keypath = await this.getBucketFolder(tenant+':'+subproject);
// tslint:disable-next-line:triple-equals
if(readonly ) { // readOnly True
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/download-role-arn')
flagUpload = false;
} else // readOnly False
{
roleArn = await this.awsSSMHelper.getSSMParameter('/osdu/' + AWSConfig.AWS_ENVIRONMENT + '/seismic-store/iam/upload-role-arn')
flagUpload = true;
}
credentials = await this.awsSTSHelper.getCredentials(s3bucket,keypath,roleArn,flagUpload,expDuration);
const result = {
access_token: credentials,
expires_in: 3599,
token_type: 'Bearer',
};
return result;
}
// this will return serviceprincipal access token
......
......@@ -18,3 +18,5 @@ export { AWSCredentials } from './credentials';
export { AWSDynamoDbDAO, AWSDynamoDbTransactionDAO, AWSDynamoDbQuery } from './dynamodb';
export { AwsTrace } from './trace';
export { AWSDataEcosystemServices } from './dataecosystem';
export { AwsSeistore } from './seistore';
export { AwsLogger } from './logger';
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { getLogger } from 'log4js';
import {AbstractLogger, LoggerFactory} from '../../logger';
import { AWSConfig } from './config';
// fetch logger and export
@LoggerFactory.register('aws')
export class AwsLogger extends AbstractLogger {
public info(data: any): void {
logger.info(data);
}
public debug(data: any): void {
logger.debug(data);
}
public error(data: any): void {
logger.error(data);
}
public metric(key:string,data: any): void {
logger.info("No Metric");
}
}
export const logger = getLogger();
export function config()
{
logger.level = AWSConfig.LOGGER_LEVEL;
}
\ No newline at end of file
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { SubProjectModel } from '../../../services/subproject';
import { AbstractSeistore, SeistoreFactory } from '../../seistore';
@SeistoreFactory.register('aws')
export class AwsSeistore extends AbstractSeistore {
public checkExtraSubprojectCreateParams(requestBody: any, subproject: SubProjectModel) { return; }
}
......@@ -15,6 +15,7 @@
// ============================================================================
import { Config, TraceFactory, ConfigFactory } from '../../..';
import { StorageJobManager } from '../../../../cloud/shared/queue';
import { Locker } from '../../../../services/dataset/locker';
import { Feature, FeatureFlags } from '../../../../shared';
......@@ -34,6 +35,13 @@ async function ServerStart() {
console.log('- Initializing redis cache')
await Locker.init();
// tslint:disable-next-line
console.log('- Initializing storage transfer deamon')
StorageJobManager.setup({
ADDRESS: Config.DES_REDIS_INSTANCE_ADDRESS,
PORT: Config.DES_REDIS_INSTANCE_PORT
})
if(FeatureFlags.isEnabled(Feature.TRACE)) {
// tslint:disable-next-line
console.log('- Initializing cloud tracer')
......
......@@ -20,7 +20,7 @@ import express from 'express';
import jwtProxy, { JwtProxyOptions } from 'jwtproxy';
import { Config, LoggerFactory } from '../../..';
import { ServiceRouter } from '../../../../services';
import { Feature, FeatureFlags, TraceLog } from '../../../../shared';
import { Feature, FeatureFlags} from '../../../../shared';
import fs from 'fs';
import https from 'https';
......@@ -75,14 +75,18 @@ export class Server {
this.app.options('*', cors());
this.app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
// create and start a new trace object
res.locals.trace = new TraceLog(req.method + ':' + req.url);
// not required anymore - to verify
if (req.get('slb-on-behalf-of') !== undefined) {
req.headers.authorization = req.get('slb-on-behalf-of');
}
// track caller to the main log
const key = req.headers['x-api-key'] as string;
const logger = LoggerFactory.build(Config.CLOUDPROVIDER);
logger.info(
((key && key.length > 5) ? ('[***' + key.substr(key.length - 5) + '] ') : '')
+ '[' + req.method + '] ' + req.url);
// init the metrics logger
if(FeatureFlags.isEnabled(Feature.LOGGING)) {
LoggerFactory.build(Config.CLOUDPROVIDER).metric('Request Size',
......
......@@ -57,8 +57,7 @@ export class AWSStorage extends AbstractStorage {
// Create a new bucket, for aws, create a folder with folderName
public async createBucket(
folderName: string,
location: string, storageClass: string,
adminACL: string, editorACL: string, viewerACL: string): Promise<void> {
location: string, storageClass: string): Promise<void> {
const folder = this.getFolder(folderName);
const params = {
Bucket: AWSConfig.AWS_BUCKET,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment