Commit c4a64f05 authored by Konstantin Khottchenkov's avatar Konstantin Khottchenkov
Browse files

Merge branch 'master' into feature/kkhottchenkov/base_e2e_tests_tunning

parents df0b96ff 13a3ade3
Pipeline #26222 passed with stages
in 8 minutes and 30 seconds
......@@ -27,7 +27,7 @@ include:
# build
- project: "osdu/platform/ci-cd-pipelines"
file: "build/seismic-store-service.yml"
# containerize
- project: "osdu/platform/ci-cd-pipelines"
file: "containerize/seismic-store-service.yml"
......@@ -44,4 +44,4 @@ include:
#gcp
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/gcp-seismic-store-service.yml"
\ No newline at end of file
file: "cloud-providers/gcp-seismic-store-service.yml"
import os
import requests
import json
url = os.getenv('KEYCLOAK_URL')
client_id = os.getenv('KEYCLOAK_CLIENT_ID')
client_secret = os.getenv('KEYCLOAK_CLIENT_SECRET')
user = os.getenv('AUTH_USER_ACCESS')
password = os.getenv('AUTH_USER_ACCESS_PASSWORD')
payload = "grant_type=password&client_id="+client_id+"&client_secret="+client_secret+"&username="+user+"&password="+password+"&scope=openid"
headers = {
'Content-Type': "application/x-www-form-urlencoded"
}
full_url="https://"+url+"/auth/realms/OSDU/protocol/openid-connect/token"
response = requests.request("POST", full_url, data=payload, headers=headers)
print(response.text)
result = response.json()
token = result['access_token']
print(token)
This diff is collapsed.
......@@ -27,10 +27,15 @@ export interface ConfigModel {
LOCKSMAP_REDIS_INSTANCE_ADDRESS: string;
LOCKSMAP_REDIS_INSTANCE_PORT: number;
LOCKSMAP_REDIS_INSTANCE_KEY?: string;
LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE?: boolean;
DES_REDIS_INSTANCE_ADDRESS: string;
DES_REDIS_INSTANCE_PORT: number;
DES_REDIS_INSTANCE_KEY?: string;
DES_SERVICE_HOST: string;
DES_REDIS_INSTANCE_TLS_DISABLE?: boolean;
DES_SERVICE_HOST_ENTITLEMENT: string;
DES_SERVICE_HOST_COMPLIANCE: string;
DES_SERVICE_HOST_STORAGE: string;
DES_SERVICE_HOST_PARTITION: string;
DES_SERVICE_APPKEY: string;
DES_GROUP_CHAR_LIMIT: number;
JWKS_URL: string;
......@@ -87,14 +92,19 @@ export abstract class Config implements IConfig {
public static LOCKSMAP_REDIS_INSTANCE_ADDRESS: string;
public static LOCKSMAP_REDIS_INSTANCE_PORT: number;
public static LOCKSMAP_REDIS_INSTANCE_KEY: string;
public static LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE: boolean;
// Redis cache for DataEcosystem results
public static DES_REDIS_INSTANCE_ADDRESS: string;
public static DES_REDIS_INSTANCE_PORT: number;
public static DES_REDIS_INSTANCE_KEY: string;
public static DES_REDIS_INSTANCE_TLS_DISABLE: boolean;
// DataEcosystem Configuration
public static DES_SERVICE_HOST: string;
public static DES_SERVICE_HOST_ENTITLEMENT: string;
public static DES_SERVICE_HOST_COMPLIANCE: string;
public static DES_SERVICE_HOST_STORAGE: string;
public static DES_SERVICE_HOST_PARTITION: string;
public static DES_SERVICE_APPKEY: string;
public static DES_GROUP_CHAR_LIMIT: number;
public static DE_FORWARD_APPKEY = Symbol('seismic-dms-fw-caller-appkey');
......@@ -136,6 +146,7 @@ export abstract class Config implements IConfig {
Config.LOCKSMAP_REDIS_INSTANCE_ADDRESS = model.LOCKSMAP_REDIS_INSTANCE_ADDRESS;
Config.LOCKSMAP_REDIS_INSTANCE_PORT = model.LOCKSMAP_REDIS_INSTANCE_PORT;
Config.LOCKSMAP_REDIS_INSTANCE_KEY = model.LOCKSMAP_REDIS_INSTANCE_KEY;
Config.LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE = model.LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE || false;
Config.DES_REDIS_INSTANCE_ADDRESS =
model.DES_REDIS_INSTANCE_ADDRESS || model.LOCKSMAP_REDIS_INSTANCE_ADDRESS;
......@@ -143,6 +154,8 @@ export abstract class Config implements IConfig {
model.DES_REDIS_INSTANCE_PORT || model.LOCKSMAP_REDIS_INSTANCE_PORT;
Config.DES_REDIS_INSTANCE_KEY =
model.DES_REDIS_INSTANCE_KEY || model.LOCKSMAP_REDIS_INSTANCE_KEY;
Config.DES_REDIS_INSTANCE_TLS_DISABLE =
model.DES_REDIS_INSTANCE_TLS_DISABLE || model.LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE;
Config.FEATURE_FLAG_AUTHORIZATION = model.FEATURE_FLAG_AUTHORIZATION;
Config.FEATURE_FLAG_LEGALTAG = model.FEATURE_FLAG_LEGALTAG;
......@@ -153,7 +166,10 @@ export abstract class Config implements IConfig {
Config.FEATURE_FLAG_LOGGING = model.FEATURE_FLAG_LOGGING;
Config.FEATURE_FLAG_STACKDRIVER_EXPORTER = model.FEATURE_FLAG_STACKDRIVER_EXPORTER;
Config.DES_SERVICE_HOST = model.DES_SERVICE_HOST;
Config.DES_SERVICE_HOST_ENTITLEMENT = model.DES_SERVICE_HOST_ENTITLEMENT;
Config.DES_SERVICE_HOST_COMPLIANCE = model.DES_SERVICE_HOST_COMPLIANCE;
Config.DES_SERVICE_HOST_STORAGE = model.DES_SERVICE_HOST_STORAGE;
Config.DES_SERVICE_HOST_PARTITION = model.DES_SERVICE_HOST_PARTITION;
Config.DES_SERVICE_APPKEY = model.DES_SERVICE_APPKEY;
Config.DES_GROUP_CHAR_LIMIT = model.DES_GROUP_CHAR_LIMIT;
......@@ -169,7 +185,10 @@ export abstract class Config implements IConfig {
Config.checkRequiredConfig(Config.CLOUDPROVIDER, 'CLOUDPROVIDER');
Config.checkRequiredConfig(Config.SERVICE_ENV, 'SERVICE_ENV');
Config.checkRequiredConfig(Config.IMP_SERVICE_ACCOUNT_SIGNER, 'IMP_SERVICE_ACCOUNT_SIGNER');
Config.checkRequiredConfig(Config.DES_SERVICE_HOST, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(Config.DES_SERVICE_HOST_ENTITLEMENT, 'DES_SERVICE_HOST_ENTITLEMENT');
Config.checkRequiredConfig(Config.DES_SERVICE_HOST_COMPLIANCE, 'DES_SERVICE_HOST_COMPLIANCE');
Config.checkRequiredConfig(Config.DES_SERVICE_HOST_STORAGE, 'DES_SERVICE_HOST_STORAGE');
Config.checkRequiredConfig(Config.DES_SERVICE_HOST_PARTITION, 'DES_SERVICE_HOST_PARTITION');
Config.checkRequiredConfig(Config.DES_SERVICE_APPKEY, 'DES_SERVICE_APPKEY');
// JWT validation
......@@ -203,4 +222,4 @@ export class ConfigFactory extends CloudFactory {
}
// Set the Utest flag correctly as sooon as the config class get loaded
Config.UTEST = process.env.UTEST;
\ No newline at end of file
Config.UTEST = process.env.UTEST;
......@@ -48,9 +48,15 @@ export class AzureConfig extends Config {
await Keyvault.loadSecrets(Keyvault.CreateSecretClient());
// data ecosystem host url and appkey
AzureConfig.DES_SERVICE_HOST = process.env.DES_SERVICE_HOST
AzureConfig.DES_SERVICE_HOST_COMPLIANCE = process.env.DES_SERVICE_HOST
AzureConfig.DES_SERVICE_HOST_ENTITLEMENT = process.env.DES_SERVICE_HOST
AzureConfig.DES_SERVICE_HOST_STORAGE = process.env.DES_SERVICE_HOST
AzureConfig.DES_SERVICE_HOST_PARTITION = process.env.DES_SERVICE_HOST
AzureConfig.DES_SERVICE_APPKEY = 'undefined'
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_HOST, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_HOST_COMPLIANCE, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_HOST_ENTITLEMENT, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_HOST_STORAGE, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_HOST_PARTITION, 'DES_SERVICE_HOST');
Config.checkRequiredConfig(AzureConfig.DES_SERVICE_APPKEY, 'DES_SERVICE_APPKEY');
// the email of the service identity used to sign an impersonation token
......@@ -74,7 +80,10 @@ export class AzureConfig extends Config {
DES_REDIS_INSTANCE_ADDRESS: AzureConfig.DES_REDIS_INSTANCE_ADDRESS,
DES_REDIS_INSTANCE_PORT: AzureConfig.DES_REDIS_INSTANCE_PORT,
DES_REDIS_INSTANCE_KEY: AzureConfig.DES_REDIS_INSTANCE_KEY,
DES_SERVICE_HOST: AzureConfig.DES_SERVICE_HOST,
DES_SERVICE_HOST_COMPLIANCE: AzureConfig.DES_SERVICE_HOST_COMPLIANCE,
DES_SERVICE_HOST_ENTITLEMENT: AzureConfig.DES_SERVICE_HOST_ENTITLEMENT,
DES_SERVICE_HOST_STORAGE: AzureConfig.DES_SERVICE_HOST_STORAGE,
DES_SERVICE_HOST_PARTITION: AzureConfig.DES_SERVICE_HOST_PARTITION,
DES_SERVICE_APPKEY: AzureConfig.DES_SERVICE_APPKEY,
DES_GROUP_CHAR_LIMIT: AzureConfig.DES_GROUP_CHAR_LIMIT,
JWKS_URL: process.env.JWKS_URL,
......
......@@ -97,7 +97,7 @@ export class AzureDataEcosystemServices extends AbstractDataEcosystemCore {
AzureConfig.SP_TENANT_ID, AzureConfig.SP_APP_RESOURCE_ID)).access_token,
'Content-Type': 'application/json'
},
url: AzureConfig.DES_SERVICE_HOST + '/api/partition/v1/partitions/' + dataPartitionID
url: AzureConfig.DES_SERVICE_HOST_PARTITION + '/api/partition/v1/partitions/' + dataPartitionID
};
try {
return JSON.parse(await request.get(options));
......@@ -112,7 +112,8 @@ export class AzureDataEcosystemServices extends AbstractDataEcosystemCore {
this._storageConfigs = new Cache<string>({
ADDRESS: AzureConfig.DES_REDIS_INSTANCE_ADDRESS,
PORT: AzureConfig.DES_REDIS_INSTANCE_PORT,
KEY: AzureConfig.DES_REDIS_INSTANCE_KEY
KEY: AzureConfig.DES_REDIS_INSTANCE_KEY,
DISABLE_TLS: AzureConfig.DES_REDIS_INSTANCE_TLS_DISABLE,
}, 'storage')
}
......@@ -137,7 +138,8 @@ export class AzureDataEcosystemServices extends AbstractDataEcosystemCore {
this._cosmosConfigs = new Cache<string>({
ADDRESS: AzureConfig.DES_REDIS_INSTANCE_ADDRESS,
PORT: AzureConfig.DES_REDIS_INSTANCE_PORT,
KEY: AzureConfig.DES_REDIS_INSTANCE_KEY
KEY: AzureConfig.DES_REDIS_INSTANCE_KEY,
DISABLE_TLS: AzureConfig.DES_REDIS_INSTANCE_TLS_DISABLE,
}, 'cosmos')
}
......
......@@ -83,7 +83,10 @@ export class ConfigGoogle extends Config {
DES_REDIS_INSTANCE_ADDRESS: process.env.DES_REDIS_INSTANCE_ADDRESS,
DES_REDIS_INSTANCE_PORT: +process.env.DES_REDIS_INSTANCE_PORT,
DES_REDIS_INSTANCE_KEY: process.env.DES_REDIS_INSTANCE_KEY,
DES_SERVICE_HOST: process.env.SEISTORE_DES_HOST,
DES_SERVICE_HOST_COMPLIANCE: process.env.SEISTORE_DES_HOST,
DES_SERVICE_HOST_ENTITLEMENT: process.env.SEISTORE_DES_HOST,
DES_SERVICE_HOST_STORAGE: process.env.SEISTORE_DES_HOST,
DES_SERVICE_HOST_PARTITION: process.env.SEISTORE_DES_HOST,
DES_SERVICE_APPKEY: process.env.SEISTORE_DES_APPKEY,
DES_GROUP_CHAR_LIMIT: ConfigGoogle.DES_GROUP_CHAR_LIMIT,
JWKS_URL: process.env.JWKS_URL,
......
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
import { Config, ConfigFactory } from '../../config';
import { config } from './logger';
@ConfigFactory.register('ibm')
export class IbmConfig extends Config {
// Apis base url path
public static API_VERSION = 'v3';
public static API_BASE_URL_PATH = '/api/' + IbmConfig.API_VERSION;
// max len for a group name in DE
public static DES_GROUP_CHAR_LIMIT = 256;
// IBM COS
public static COS_ACCESS_KEY_ID: string;
public static COS_SECRET_ACCESS_KEY: string;
public static COS_ENDPOINT: string;
public static COS_S3_FORCEPATHSTYLE: boolean;
public static COS_SIGNATUREVERSION: string;
// IBM Keycloak
public static KEYCLOAK_BASEURL: string;
public static KEYCLOAK_URL_TOKEN: string;
public static KEYCLOAK_REALM: string;
public static KEYCLOAK_USERNAME: string;
public static KEYCLOAK_PASSWORD: string;
public static KEYCLOAK_GRANTTYPE: string;
public static KEYCLOAK_CLIENTID: string;
public static KEYCLOAK_CLIENTSECRET: string;
//IBM Document DB
public static DOC_DB_URL: string;
public static DOC_DB_COLLECTION: string;
public static DOC_DB_QUERY_RESULT_LIMIT: string;
public static DOC_DB_QUERY_RESULT_LIMIT_VALUE: number;
public static DOC_DB_QUERY_SELECT_FIELDS: string;
// SERVICES
public static ENTITLEMENT_HOST;
public static LEGAL_HOST;
public static STORAGE_HOST;
//Logger
public static LOGGER_LEVEL;
public async init(): Promise<void> {
// data ecosystem host url and appkey
IbmConfig.DES_SERVICE_HOST_COMPLIANCE = process.env.DES_SERVICE_HOST_COMPLIANCE;
IbmConfig.DES_SERVICE_HOST_ENTITLEMENT = process.env.DES_SERVICE_HOST_ENTITLEMENT;
IbmConfig.DES_SERVICE_HOST_STORAGE = process.env.DES_SERVICE_HOST_STORAGE;
IbmConfig.IMP_SERVICE_ACCOUNT_SIGNER = process.env.IMP_SERVICE_ACCOUNT_SIGNER;
IbmConfig.ENTITLEMENT_HOST = process.env.ENTITLEMENT_HOST;//DES_SERVICE_HOST replaced by new variable ENTITLEMENT_HOST
IbmConfig.LEGAL_HOST = process.env.LEGAL_HOST;//DES_SERVICE_HOST replaced by new variable LEGAL_HOST
IbmConfig.STORAGE_HOST = process.env.STORAGE_HOST;//DES_SERVICE_HOST replaced by new variable LEGAL_HOST
// IBM COS
IbmConfig.COS_ACCESS_KEY_ID = process.env.COS_ACCESS_KEY_ID;
IbmConfig.COS_SECRET_ACCESS_KEY = process.env.COS_SECRET_ACCESS_KEY;
IbmConfig.COS_ENDPOINT = process.env.COS_ENDPOINT;
IbmConfig.COS_S3_FORCEPATHSTYLE = process.env.COS_S3_FORCEPATHSTYLE === 'true';//string to boolean
IbmConfig.COS_SIGNATUREVERSION = process.env.COS_SIGNATUREVERSION;
// IBM Keycloak
IbmConfig.KEYCLOAK_BASEURL = process.env.KEYCLOAK_BASEURL;
IbmConfig.KEYCLOAK_URL_TOKEN = process.env.KEYCLOAK_URL_TOKEN;
IbmConfig.KEYCLOAK_USERNAME = process.env.KEYCLOAK_USERNAME;
IbmConfig.KEYCLOAK_PASSWORD = process.env.KEYCLOAK_PASSWORD;
IbmConfig.KEYCLOAK_GRANTTYPE = process.env.KEYCLOAK_GRANTTYPE;
IbmConfig.KEYCLOAK_CLIENTID = process.env.KEYCLOAK_CLIENTID;
IbmConfig.KEYCLOAK_CLIENTSECRET = process.env.KEYCLOAK_CLIENTSECRET;
IbmConfig.KEYCLOAK_REALM = process.env.KEYCLOAK_REALM;
//IBM Document DB
IbmConfig.DOC_DB_URL = process.env.DB_URL;
IbmConfig.DOC_DB_COLLECTION = process.env.DOC_DB_COLLECTION;
IbmConfig.DOC_DB_QUERY_SELECT_FIELDS = process.env.DOC_DB_QUERY_SELECT_FIELDS;
IbmConfig.DOC_DB_QUERY_RESULT_LIMIT = process.env.DOC_DB_QUERY_RESULT_LIMIT;
IbmConfig.DOC_DB_QUERY_RESULT_LIMIT_VALUE = parseInt(process.env.DOC_DB_QUERY_RESULT_LIMIT_VALUE, 10);
///////////////////////////////////////
IbmConfig.DES_SERVICE_APPKEY = 'na'
Config.checkRequiredConfig(IbmConfig.DES_SERVICE_HOST_COMPLIANCE, 'DES_SERVICE_HOST_COMPLIANCE');
Config.checkRequiredConfig(IbmConfig.DES_SERVICE_HOST_ENTITLEMENT, 'DES_SERVICE_HOST_ENTITLEMENT');
Config.checkRequiredConfig(IbmConfig.DES_SERVICE_HOST_STORAGE, 'DES_SERVICE_HOST_STORAGE');
Config.checkRequiredConfig(IbmConfig.DES_SERVICE_APPKEY, 'DES_SERVICE_APPKEY');
// redis cache port for locks (the port as env variable)
IbmConfig.LOCKSMAP_REDIS_INSTANCE_PORT = +process.env.REDIS_INSTANCE_PORT
IbmConfig.LOCKSMAP_REDIS_INSTANCE_ADDRESS = process.env.LOCKSMAP_REDIS_INSTANCE_ADDRESS
IbmConfig.LOCKSMAP_REDIS_INSTANCE_KEY = process.env.LOCKSMAP_REDIS_INSTANCE_KEY
IbmConfig.LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE = process.env.CACHE_TLS_DISABLE ? true : false
IbmConfig.DES_REDIS_INSTANCE_ADDRESS = process.env.DES_REDIS_INSTANCE_ADDRESS
IbmConfig.DES_REDIS_INSTANCE_PORT = +process.env.DES_REDIS_INSTANCE_PORT
IbmConfig.DES_REDIS_INSTANCE_KEY = process.env.DES_REDIS_INSTANCE_KEY
IbmConfig.DES_REDIS_INSTANCE_TLS_DISABLE = process.env.CACHE_TLS_DISABLE ? true : false
//Logger
IbmConfig.LOGGER_LEVEL = process.env.LOGGER_LEVEL || 'debug';
Config.checkRequiredConfig(IbmConfig.LOCKSMAP_REDIS_INSTANCE_PORT, 'REDIS_INSTANCE_PORT');
// init generic configurations
await Config.initServiceConfiguration({
SERVICE_ENV: process.env.APP_ENVIRONMENT_IDENTIFIER,
SERVICE_PORT: +process.env.PORT || 5000,
API_BASE_PATH: IbmConfig.API_BASE_URL_PATH,
IMP_SERVICE_ACCOUNT_SIGNER: IbmConfig.IMP_SERVICE_ACCOUNT_SIGNER,
LOCKSMAP_REDIS_INSTANCE_ADDRESS: IbmConfig.LOCKSMAP_REDIS_INSTANCE_ADDRESS,
LOCKSMAP_REDIS_INSTANCE_PORT: IbmConfig.LOCKSMAP_REDIS_INSTANCE_PORT,
LOCKSMAP_REDIS_INSTANCE_KEY: IbmConfig.LOCKSMAP_REDIS_INSTANCE_KEY,
LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE: IbmConfig.LOCKSMAP_REDIS_INSTANCE_TLS_DISABLE,
DES_REDIS_INSTANCE_ADDRESS: IbmConfig.DES_REDIS_INSTANCE_ADDRESS,
DES_REDIS_INSTANCE_PORT: IbmConfig.DES_REDIS_INSTANCE_PORT,
DES_REDIS_INSTANCE_KEY: IbmConfig.DES_REDIS_INSTANCE_KEY,
DES_REDIS_INSTANCE_TLS_DISABLE: IbmConfig.DES_REDIS_INSTANCE_TLS_DISABLE,
DES_SERVICE_HOST_COMPLIANCE: IbmConfig.DES_SERVICE_HOST_COMPLIANCE,
DES_SERVICE_HOST_ENTITLEMENT: IbmConfig.DES_SERVICE_HOST_ENTITLEMENT,
DES_SERVICE_HOST_STORAGE: IbmConfig.DES_SERVICE_HOST_STORAGE,
DES_SERVICE_HOST_PARTITION: 'TO DEFINE',
DES_SERVICE_APPKEY: IbmConfig.DES_SERVICE_APPKEY,
DES_GROUP_CHAR_LIMIT: IbmConfig.DES_GROUP_CHAR_LIMIT,
JWKS_URL: process.env.JWKS_URL,
JWT_EXCLUDE_PATHS: process.env.JWT_EXCLUDE_PATHS,
JWT_AUDIENCE: process.env.JWT_AUDIENCE,
JWT_ENABLE_FEATURE: process.env.JWT_ENABLE_FEATURE ? process.env.JWT_ENABLE_FEATURE === 'true' : false,
TENANT_JOURNAL_ON_DATA_PARTITION: true,
FEATURE_FLAG_AUTHORIZATION: process.env.FEATURE_FLAG_AUTHORIZATION !== undefined ?
process.env.FEATURE_FLAG_AUTHORIZATION !== 'false' : true,
FEATURE_FLAG_LEGALTAG: process.env.FEATURE_FLAG_LEGALTAG !== undefined ?
process.env.FEATURE_FLAG_LEGALTAG !== 'false' : true,
FEATURE_FLAG_SEISMICMETA_STORAGE: process.env.FEATURE_FLAG_SEISMICMETA_STORAGE !== undefined ?
process.env.FEATURE_FLAG_SEISMICMETA_STORAGE !== 'false' : true,
FEATURE_FLAG_IMPTOKEN: process.env.FEATURE_FLAG_IMPTOKEN !== undefined ?
process.env.FEATURE_FLAG_IMPTOKEN !== 'false' : true,
FEATURE_FLAG_STORAGE_CREDENTIALS: process.env.FEATURE_FLAG_STORAGE_CREDENTIALS !== undefined ?
process.env.FEATURE_FLAG_STORAGE_CREDENTIALS !== 'false' : true,
FEATURE_FLAG_TRACE: process.env.FEATURE_FLAG_TRACE !== undefined ?
process.env.FEATURE_FLAG_TRACE !== 'false' : true,
FEATURE_FLAG_LOGGING: process.env.FEATURE_FLAG_LOGGING !== undefined ?
process.env.FEATURE_FLAG_LOGGING !== 'false' : true,
FEATURE_FLAG_STACKDRIVER_EXPORTER: process.env.FEATURE_FLAG_STACKDRIVER_EXPORTER !== undefined ?
process.env.FEATURE_FLAG_STACKDRIVER_EXPORTER !== 'false' : true,
});
config();
}
}
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
import { AbstractStorage, StorageFactory } from '../../storage';
import AWS from 'aws-sdk';
import { Config } from '../../config';
import { IbmConfig } from './config';
import { logger } from './logger';
let cosStorage;
@StorageFactory.register('ibm')
export class Cos extends AbstractStorage {
private COS_SUBPROJECT_BUCKET_PREFIX = 'ss-' + Config.SERVICE_ENV;
public constructor() {
super();
logger.info('In Cos.constructor. Instantiating cos client.');
cosStorage = new AWS.S3({
accessKeyId: IbmConfig.COS_ACCESS_KEY_ID ,
secretAccessKey: IbmConfig.COS_SECRET_ACCESS_KEY ,
endpoint: IbmConfig.COS_ENDPOINT,
s3ForcePathStyle: IbmConfig.COS_S3_FORCEPATHSTYLE, // needed with minio?
signatureVersion: IbmConfig.COS_SIGNATUREVERSION
});
}
//generate a random bucket name
public randomBucketName(): string {
logger.info('In Cos.randomBucketName.');
let suffix = Math.random().toString(36).substring(2, 16);
suffix = suffix + Math.random().toString(36).substring(2, 16);
suffix = suffix.substr(0, 16);
logger.info('Returning from Cos.randomBucketName. BucketName - '+this.COS_SUBPROJECT_BUCKET_PREFIX + '-' + suffix);
return this.COS_SUBPROJECT_BUCKET_PREFIX + '-' + suffix;
}
//Create a new bucket
public async createBucket(
bucketName: string,
location: string, storageClass: string,
adminACL: string, editorACL: string, viewerACL: string): Promise<void> {
logger.info('In Cos.createBucket.');
///not sure how to use ACLs
const bucketParams = {
Bucket: bucketName,
CreateBucketConfiguration: {
// Set your region here
// LocationConstraint: location
}
};
cosStorage.createBucket(bucketParams, function(err, data) {
if (err) {
logger.error('Error while creating bucket. Error stack - '+ err.stack);
throw err;
}
else
logger.info('Bucket created successfully @'+ data.Location);
});
logger.info('Returning from Cos.createBucket.');
}
//Cos bucket deletion
public async deleteBucket(bucketName: string, force = false): Promise<void> {
logger.info('In Cos.deleteBucket.');
var params = {Bucket: bucketName};
cosStorage.deleteBucket(params, function(err) {
if (err) {
logger.error('Unable to delete bucket. Error stack'+ err.stack);
throw err;
}
logger.info('Removed bucket.');
})
logger.info('Returning from Cos.deleteBucket.');
}
//Deletion of files in Cos bucket
public async deleteFiles(bucketName: string): Promise<void> {
logger.info('In Cos.deleteFiles.');
var self = this;
cosStorage.listObjects({Bucket: bucketName}, function (err, data) {
if (err) {
logger.error("error listing bucket objects "+err);
throw err;
}
var items = data.Contents;
if(!items || items.length<=0)
logger.info('No items to delete.');
else
for (var i = 0; i < items.length; i += 1) {
var objectKey = items[i].Key;
logger.info('Object to be deleted. objectKey-'+objectKey);
self.deleteObject(bucketName, objectKey);
}
});
logger.info('Returning from Cos.deleteFiles.');
}
//Saving file in Cos bucket
public async saveObject(bucketName: string, objectName: string, data: string): Promise<void> {
logger.info('In Cos.saveObject.');
let params = {Bucket: bucketName, Key: objectName, Body: data};
cosStorage.putObject(params, function(err, data) {
if (err) {
logger.error('Object not saved.'+err);
throw err;
}
else
logger.debug("Object saved."+ data);
});
logger.info('Returning from Cos.saveObject.');
}
//delete an object from a bucket
public async deleteObject(bucketName: string, objectName: string): Promise<void> {
///used to delete CDO file
logger.info('In Cos.deleteObject.');
let params = {Bucket: bucketName, Key: objectName};
cosStorage.deleteObject(params, function(err) {
if (err) {
logger.error('Unable to remove object'+ err);
throw err;
}
logger.info('Removed the object')
});
logger.info('Returning from Cos.deleteObject.');
}
//delete multiple objects
public async deleteObjects(bucketName: string, prefix: string, async: boolean = false): Promise<void> {
logger.info('not sure of the use of this function');
logger.info('Returning from Cos.deleteObject.');
await Promise.resolve();
}
//copy multiple objects (skip the dummy file)
///implemention aws sdk copyObject to copy dataset
public async copy(bucketIn: string, prefixIn: string, bucketOut: string, prefixOut: string, ownerEmail: string): Promise<void> {
///const res = 'In copy: bucketIn - '+bucketIn + ' prefixIn - '+prefixIn+ ' bucketOut - '+bucketOut + ' prefixOut - '+prefixOut;
logger.info('In Cos.copy.');
// Create the parameters for calling listObjects
var bucketParams = {
Bucket : bucketIn,
};
// Call S3 to obtain a list of the objects in the bucket
cosStorage.listObjects(bucketParams, function(err, data) {
if (err) {
logger.error("Error in listing objects."+ err);
throw err;
} else {
logger.debug("List of objects. Data-"+ data);
}
});
let params = {
Bucket : bucketIn+'/'+prefixIn, /* Target required */
CopySource : bucketOut+'/'+prefixOut, /* Source required */
Key : "*", /* What is required */
//ACL : 'public-read',
};
cosStorage.copyObject(params, function(err, data) {
if (err) {
logger.error('Error while copying object. Error stack - '+err);
throw err;
}
else {
logger.info('Object copied. Data - '+data);
}
});
logger.info('Returning from Cos.deleteObject.');
}
//check bucket exists or not
public async bucketExists(bucketName: string): Promise<boolean> {
//const result = await cosStorage.bucket(bucketName).exists();
logger.info('In Cos.bucketExists.');
const bucketParams = {
Bucket: bucketName
};
try {
const result = await cosStorage.headBucket(bucketParams).promise();
logger.info('Bucket exists. Returning from Cos.bucketExists.');
return true;
}
catch(err) {
logger.error('Bucket does not exist. Error stack - '+err.stack);
if (err.statusCode === 404) {
return false;
}
throw err;
}
}