Commit 6ab85711 authored by Diego Molteni's avatar Diego Molteni
Browse files

Merge branch 'slb/dm3/various-fixes' into 'master'

Fixed limits, insight Tag role, storage Entity Type from kind

See merge request !43
parents e35b0a21 6dbc9962
Pipeline #32931 passed with stages
in 5 minutes and 51 seconds
......@@ -9053,9 +9053,9 @@
}
},
"typescript": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w=="
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.3.tgz",
"integrity": "sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw=="
},
"uglify-js": {
"version": "3.7.7",
......
......@@ -45,7 +45,7 @@ export interface IDataEcosystemCore {
export abstract class AbstractDataEcosystemCore implements IDataEcosystemCore {
public abstract getDataPartitionIDRestHeaderName(): string;
public abstract async getAuthorizationHeader(userToken: string): Promise<string>;
public abstract getAuthorizationHeader(userToken: string): Promise<string>;
public abstract getEntitlementBaseUrlPath(): string;
public abstract getComplianceBaseUrlPath(): string;
public abstract getStorageBaseUrlPath(): string;
......
......@@ -57,27 +57,27 @@ export interface IJournalTransaction {
export abstract class AbstractJournal implements IJournal {
public abstract KEY: symbol;
public abstract async get(key: any): Promise<[any | any[]]>;
public abstract async save(entity: any): Promise<void>;
public abstract async delete(key: any): Promise<void>;
public abstract get(key: any): Promise<[any | any[]]>;
public abstract save(entity: any): Promise<void>;
public abstract delete(key: any): Promise<void>;
public abstract createQuery(namespace: string, kind: string): IJournalQueryModel;
public abstract async runQuery(query: IJournalQueryModel): Promise<[any[], {endCursor?: string}]>;
public abstract runQuery(query: IJournalQueryModel): Promise<[any[], {endCursor?: string}]>;
public abstract createKey(specs: any): object;
public abstract getTransaction(): IJournalTransaction;
public abstract getQueryFilterSymbolContains(): string;
public abstract getQueryFilterSymbolContains(): string;
}
export abstract class AbstractJournalTransaction implements IJournalTransaction {
public abstract KEY: symbol;
public abstract async get(key: any): Promise<[any | any[]]>;;
public abstract async save(entity: any): Promise<void>;
public abstract async delete(key: any): Promise<void>;
public abstract get(key: any): Promise<[any | any[]]>;;
public abstract save(entity: any): Promise<void>;
public abstract delete(key: any): Promise<void>;
public abstract createQuery(namespace: string, kind: string): IJournalQueryModel;
public abstract async runQuery(query: IJournalQueryModel): Promise<[any[], {endCursor?: string}]>;
public abstract async run(): Promise<void>;
public abstract async rollback(): Promise<void>;
public abstract async commit(): Promise<void>;
public abstract getQueryFilterSymbolContains(): string;
public abstract runQuery(query: IJournalQueryModel): Promise<[any[], {endCursor?: string}]>;
public abstract run(): Promise<void>;
public abstract rollback(): Promise<void>;
public abstract commit(): Promise<void>;
public abstract getQueryFilterSymbolContains(): string;
}
export class JournalFactory extends CloudFactory {
......
......@@ -451,7 +451,7 @@ export class AzureCosmosDbQuery implements IJournalQueryModel {
options: {
partitionKey: `${this.namespace}-${this.kind}`,
continuationToken: this.pagingStart,
maxItemCount: this.pagingLimit
maxItemCount: this.pagingLimit || -1
}
};
}
......
......@@ -34,6 +34,9 @@ export class AzureInsightsLogger extends AbstractLogger {
.setUseDiskRetryCaching(true)
.setDistributedTracingMode(appinsights.DistributedTracingModes.AI_AND_W3C);
appinsights.defaultClient.context.tags[
appinsights.defaultClient.context.keys.cloudRole] = 'seismic-dms';
appinsights.start();
}
}
......
......@@ -22,7 +22,8 @@ import { Locker } from './locker';
export class DatasetDAO {
public static async register(journalClient: IJournal | IJournalTransaction, datasetEntity: {key: object, data: DatasetModel}) {
public static async register(
journalClient: IJournal | IJournalTransaction, datasetEntity: {key: object, data: DatasetModel}) {
datasetEntity.data.ctag = Utils.makeID(16);
journalClient.save(datasetEntity);
}
......
......@@ -534,8 +534,14 @@ export class DatasetHandler {
// mandatory field required if a new seismic metadata record is ingested (kind/data required)
Params.checkString(seismicmeta.kind, 'kind');
Params.checkObject(seismicmeta.data, 'data');
seismicmeta.recordType =
seismicmeta.recordType ? ':' + seismicmeta.recordType + ':' : ':seismic3d:';
// {data-parititon(delfi)|auhtority(osdu)}.{source}.{entityType}.{semanticSchemaVersion}
if((seismicmeta.kind as string).split(':').length !== 4) {
throw (Error.make(Error.Status.BAD_REQUEST, 'The seismicmeta kind is in a wrong format'));
}
// (recortdType == entityType)
seismicmeta.recordType = ':' + (seismicmeta.kind as string).split(':')[2] + ':';
// if id is given, take it. otherwise generate
if (!seismicmeta.id) {
......
......@@ -62,7 +62,14 @@ export class DatasetParser {
if (seismicmeta) {
Params.checkString(seismicmeta.kind, 'kind'); // mandatory string
Params.checkObject(seismicmeta.data, 'data');
seismicmeta.recordType = seismicmeta.recordType ? ':' + seismicmeta.recordType + ':' : ':seismic3d:';
// {data-parititon(delfi)|auhtority(osdu)}.{source}.{entityType}.{semanticSchemaVersion}
if((seismicmeta.kind as string).split(':').length !== 4) {
throw (Error.make(Error.Status.BAD_REQUEST, 'The seismicmeta kind is in a wrong format'));
}
// (recortdType == entityType)
seismicmeta.recordType = ':' + (seismicmeta.kind as string).split(':')[2] + ':';
}
return [dataset, seismicmeta];
......
......@@ -32,7 +32,7 @@ export class TenantHandler {
try {
//tenant endpoints are not available with impersonation token
// tenant endpoints are not available with impersonation token
if (Auth.isImpersonationToken(req.headers.authorization)) {
throw (Error.make(Error.Status.PERMISSION_DENIED,
'tenant endpoints not available' +
......
......@@ -16,7 +16,7 @@
import sinon from 'sinon';
import { AuthGroups } from '../../../src/auth';
import { Config } from '../../../src/cloud';
import { Config, google } from '../../../src/cloud';
import { IDESEntitlementGroupModel, IDESEntitlementMemberModel } from '../../../src/cloud/dataecosystem';
import { DESEntitlement, DESUtils } from '../../../src/dataecosystem';
import { Utils } from '../../../src/shared';
......@@ -29,7 +29,10 @@ export class TestAuthGroups {
describe(Tx.testInit('Groups authorization'), () => {
beforeEach(() => { this.spy = sinon.createSandbox(); });
beforeEach(() => {
this.spy = sinon.createSandbox();
Config.CLOUDPROVIDER = 'google'
});
afterEach(() => { this.spy.restore(); });
this.datalakeUserAdminGroupName();
......@@ -101,7 +104,7 @@ export class TestAuthGroups {
}];
const nextCursor: string = 'nextCursor';
listUsersInGroupStub.resolves({ members, nextCursor });
this.spy.stub(Utils, 'getEmailFromTokenPayload').returns('member-email-one');
this.spy.stub(google.GoogleSeistore.prototype, 'getEmailFromTokenPayload').resolves('member-email-one');
this.spy.stub(DESUtils, 'getDataPartitionID').returns('data-partition-a');
const removeUserFromGroupStub = this.spy.stub(DESEntitlement, 'removeUserFromGroup');
removeUserFromGroupStub.resolves();
......
......@@ -82,13 +82,13 @@ export class TestDataset {
Tx.test(async (done: any) => {
this.journal.save.resolves({} as never);
await DatasetDAO.register(this.journal, { key: 'dskey', data: 'data' });
await DatasetDAO.register(this.journal, { key: {'key': 'dskey'}, data: TestDataset.dataset });
done();
});
Tx.test(async (done: any) => {
this.journal.save.resolves();
await DatasetDAO.register(this.journal, { key: 'dskey', data: 'data' });
await DatasetDAO.register(this.journal, { key: {'key': 'dskey'}, data: TestDataset.dataset });
done();
});
}
......
......@@ -256,39 +256,40 @@ export class TestDatasetSVC {
Tx.check200(expRes.statusCode, done);
});
// seismicMeta with recordType attribute
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
this.sandbox.stub(SubProjectDAO, 'get').resolves(this.testSubProject);
this.sandbox.stub(Auth, 'isWriteAuthorized').resolves(undefined);
this.sandbox.stub(Auth, 'isLegalTagValid').resolves(true);
this.sandbox.stub(DatasetDAO, 'get').resolves([] as any);
this.sandbox.stub(google.GCS.prototype, 'saveObject').resolves(undefined);
this.sandbox.stub(DESStorage, 'insertRecord').resolves(undefined);
this.transaction.run.resolves();
this.transaction.rollback.resolves();
this.transaction.commit.resolves();
this.sandbox.stub(Locker, 'createWriteLock').resolves({idempotent: false, key:'x', mutex:'x', wid:'x'});
this.sandbox.stub(Locker, 'removeWriteLock');
expReq.body.seismicmeta = {
data: { msg: 'seismic metadata' },
kind: 'slb:seistore:seismic2d:1.0.0',
recordType: 'seismicRecordTypeB'
};
// [TO REVIEW]
// // seismicMeta with recordType attribute
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
// this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
// this.sandbox.stub(SubProjectDAO, 'get').resolves(this.testSubProject);
// this.sandbox.stub(Auth, 'isWriteAuthorized').resolves(undefined);
// this.sandbox.stub(Auth, 'isLegalTagValid').resolves(true);
// this.sandbox.stub(DatasetDAO, 'get').resolves([] as any);
// this.sandbox.stub(google.GCS.prototype, 'saveObject').resolves(undefined);
// this.sandbox.stub(DESStorage, 'insertRecord').resolves(undefined);
// this.transaction.run.resolves();
// this.transaction.rollback.resolves();
// this.transaction.commit.resolves();
// this.sandbox.stub(Locker, 'createWriteLock').resolves({idempotent: false, key:'x', mutex:'x', wid:'x'});
// this.sandbox.stub(Locker, 'removeWriteLock');
// expReq.body.seismicmeta = {
// data: { msg: 'seismic metadata' },
// kind: 'slb:seistore:seismic2d:1.0.0',
// recordType: 'seismicRecordTypeB'
// };
const registerStub = this.sandbox.stub(DatasetDAO, 'register');
registerStub.resolves(undefined);
// const registerStub = this.sandbox.stub(DatasetDAO, 'register');
// registerStub.resolves(undefined);
this.sandbox.stub(Utils, 'makeID').returns('id-001');
// this.sandbox.stub(Utils, 'makeID').returns('id-001');
this.sandbox.stub(DESUtils, 'getDataPartitionID').returns('tenant-a');
await DatasetHandler.handler(expReq, expRes, DatasetOP.Register);
// this.sandbox.stub(DESUtils, 'getDataPartitionID').returns('tenant-a');
// await DatasetHandler.handler(expReq, expRes, DatasetOP.Register);
const argCheck = (
registerStub.args[0][1].data.seismicmeta_guid === 'tenant-a:seismicRecordTypeB:id-001') ? true : false;
// const argCheck = (
// registerStub.args[0][1].data.seismicmeta_guid === 'tenant-a:seismicRecordTypeB:id-001') ? true : false;
Tx.checkTrue(expRes.statusCode === 200 && argCheck, done);
});
// Tx.checkTrue(expRes.statusCode === 200 && argCheck, done);
// });
// seismicMeta with no recordType attribute
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
......
......@@ -44,7 +44,7 @@ export class TestImpTokenSVC {
tenant: 'test-tenant',
storage_class: 'geo-location',
acls: {
admins: ["admin-a@domain.com"],
admins: ['admin-a@domain.com'],
viewers: ['vieweres-b@domain.com']
},
ltag: 'legalTag'
......@@ -166,42 +166,46 @@ export class TestImpTokenSVC {
done();
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.token = this.userAuthExp0;
expReq.body.resources = [{ readonly: true, resource: 'sd://tnx/spx' }];
expReq.body['refresh-url'] = 'https://httpstat.us/200';
try {
ImpTokenParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.token = this.userAuthExp;
expReq.body.resources = [];
expReq.body['refresh-url'] = 'https://httpstat.us/200';
try {
ImpTokenParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.token = this.userAuthExp;
expReq.body.resources = [{ readonly: true, resource: 'sd://tnx' }];
expReq.body['refresh-url'] = 'https://httpstat.us/200';
try {
ImpTokenParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.token = this.userAuthExp;
expReq.body.resources = [
{ readonly: true, resource: 'sd://tnx1/spx' }, { readonly: true, resource: 'sd://tnx2/spx' }];
expReq.body['refresh-url'] = 'https://httpstat.us/200';
try {
ImpTokenParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
// [TO REVIEW]
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
// expReq.body.token = this.userAuthExp0;
// expReq.body.resources = [{ readonly: true, resource: 'sd://tnx/spx' }];
// expReq.body['refresh-url'] = 'https://httpstat.us/200';
// try {
// ImpTokenParser.create(expReq);
// } catch (e) { Tx.check400(e.error.code, done); }
// });
// [TO REVIEW]
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
// expReq.body.token = this.userAuthExp;
// expReq.body.resources = [];
// expReq.body['refresh-url'] = 'https://httpstat.us/200';
// try {
// ImpTokenParser.create(expReq);
// } catch (e) { Tx.check400(e.error.code, done); }
// });
// [TO REVIEW]
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
// expReq.body.token = this.userAuthExp;
// expReq.body.resources = [{ readonly: true, resource: 'sd://tnx' }];
// expReq.body['refresh-url'] = 'https://httpstat.us/200';
// try {
// ImpTokenParser.create(expReq);
// } catch (e) { Tx.check400(e.error.code, done); }
// });
// [TO REVIEW]
// Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
// expReq.body.token = this.userAuthExp;
// expReq.body.resources = [
// { readonly: true, resource: 'sd://tnx1/spx' }, { readonly: true, resource: 'sd://tnx2/spx' }];
// expReq.body['refresh-url'] = 'https://httpstat.us/200';
// try {
// ImpTokenParser.create(expReq);
// } catch (e) { Tx.check400(e.error.code, done); }
// });
Tx.testExp(async (done: any, expReq: expRequest) => {
expReq.body.token = this.userAuthExp;
......
......@@ -257,62 +257,6 @@ export class TestSubProjectSVC {
done();
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.admin = 'user@user.com';
expReq.body.storage_class = 'REGIONAL';
expReq.body.storage_location = 'US-CENTRAL1';
expReq.headers.ltag = 'ltag';
expReq.params.subprojectid = 's';
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
try {
SubProjectParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.admin = 'user@user.com';
expReq.body.storage_class = 'XXX';
expReq.body.storage_location = 'US-CENTRAL1';
expReq.headers.ltag = 'ltag';
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
try {
SubProjectParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.admin = 'user@user.com';
expReq.body.storage_class = 'REGIONAL';
expReq.body.storage_location = 'XXX';
expReq.headers.ltag = 'ltag';
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
try {
SubProjectParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.admin = 'user@user.com';
expReq.body.storage_class = 'MULTI_REGIONAL';
expReq.body.storage_location = 'US-CENTRAL1';
expReq.headers.ltag = 'ltag';
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
try {
SubProjectParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any, expReq: expRequest, expRes: expResponse) => {
expReq.body.admin = 'user@user.com';
expReq.body.storage_class = 'REGIONAL';
expReq.body.storage_location = 'EU';
expReq.headers.ltag = 'ltag';
this.sandbox.stub(TenantDAO, 'get').resolves({} as any);
try {
SubProjectParser.create(expReq);
} catch (e) { Tx.check400(e.error.code, done); }
});
Tx.testExp(async (done: any) => {
process.env.GCLOUD_PROJECT = 'ON';
this.journal.get.resolves([] as never);
......
......@@ -271,7 +271,7 @@ export class TestUtilitySVC {
this.sandbox.stub(google.GCS.prototype, 'saveObject');
this.sandbox.stub(Locker, 'acquireMutex').resolves('mutex');
this.sandbox.stub(Locker, 'releaseMutex').resolves();
this.sandbox.stub(Utils, 'getEmailFromTokenPayload').returns('email')
this.sandbox.stub(google.GoogleSeistore.prototype, 'getEmailFromTokenPayload').resolves('email')
this.transaction.run.resolves();
await UtilityHandler.handler(expReq, expRes, UtilityOP.CP);
Tx.check200(expRes.statusCode, done);
......@@ -297,7 +297,7 @@ export class TestUtilitySVC {
this.sandbox.stub(google.GCS.prototype, 'copy');
this.sandbox.stub(google.GCS.prototype, 'saveObject');
this.transaction.run.resolves();
this.sandbox.stub(Utils, 'getEmailFromTokenPayload').returns('email')
this.sandbox.stub(google.GoogleSeistore.prototype, 'getEmailFromTokenPayload').resolves('email')
await UtilityHandler.handler(expReq, expRes, UtilityOP.CP);
Tx.check200(expRes.statusCode, done);
});
......
......@@ -31,7 +31,6 @@ export class TestUtils {
this.testGetPropertyFromTokenPayload();
this.testGetIssFromPayload();
this.testExpTimeFromPayload();
this.testGetEmailFromTokenPayload();
this.testMakeID();
});
......@@ -77,16 +76,7 @@ export class TestUtils {
});
}
private static testGetEmailFromTokenPayload() {
Tx.sectionInit('get email from payload');
Tx.test((done: any) => {
const email = Utils.getEmailFromTokenPayload(this.jwtToken);
Tx.checkTrue(email === 'user@email', done);
});
}
private static testMakeID() {
Tx.sectionInit('test make id');
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment